You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by ed...@apache.org on 2022/11/15 22:44:19 UTC
[cassandra] branch trunk updated (f317eb4f2f -> d5fd0dceee)
This is an automated email from the ASF dual-hosted git repository.
edimitrova pushed a change to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra.git
from f317eb4f2f Merge branch 'cassandra-4.1' into trunk
add 444da423d5 CirleCI: Add large Dtests and J8/J11 Java distributed tests; Fix CQLSHLIB tests config and Python DTests split; CQLSHLIB tests should run with NUM_TOKENS=16 patch by Ekaterina Dimitrova; reviewed by Andres de la Pena for CASSANDRA-18001
new d5fd0dceee Merge branch 'cassandra-4.1' into trunk
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.circleci/config-2_1.yml | 397 +++++++-
.circleci/config-2_1.yml.high_res.patch | 62 +-
.circleci/config-2_1.yml.mid_res.patch | 97 +-
.circleci/config.yml | 1664 +++++++++++++++++++++++++++---
.circleci/config.yml.HIGHRES | 1666 ++++++++++++++++++++++++++++---
.circleci/config.yml.LOWRES | 1664 +++++++++++++++++++++++++++---
.circleci/config.yml.MIDRES | 1664 +++++++++++++++++++++++++++---
.circleci/generate.sh | 16 +-
.circleci/readme.md | 4 +
pylib/cassandra-cqlsh-tests.sh | 2 +-
10 files changed, 6559 insertions(+), 677 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org
[cassandra] 01/01: Merge branch 'cassandra-4.1' into trunk
Posted by ed...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
edimitrova pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra.git
commit d5fd0dceee8f9ac5ed454e781a8c319a7b1b6677
Merge: f317eb4f2f 444da423d5
Author: Ekaterina Dimitrova <ek...@datastax.com>
AuthorDate: Tue Nov 15 16:41:42 2022 -0500
Merge branch 'cassandra-4.1' into trunk
.circleci/config-2_1.yml | 397 +++++++-
.circleci/config-2_1.yml.high_res.patch | 62 +-
.circleci/config-2_1.yml.mid_res.patch | 97 +-
.circleci/config.yml | 1664 +++++++++++++++++++++++++++---
.circleci/config.yml.HIGHRES | 1666 ++++++++++++++++++++++++++++---
.circleci/config.yml.LOWRES | 1664 +++++++++++++++++++++++++++---
.circleci/config.yml.MIDRES | 1664 +++++++++++++++++++++++++++---
.circleci/generate.sh | 16 +-
.circleci/readme.md | 4 +
pylib/cassandra-cqlsh-tests.sh | 2 +-
10 files changed, 6559 insertions(+), 677 deletions(-)
diff --cc .circleci/config-2_1.yml.high_res.patch
index 56b8d0d3d9,ded21fff8c..7da8d902fb
--- a/.circleci/config-2_1.yml.high_res.patch
+++ b/.circleci/config-2_1.yml.high_res.patch
@@@ -1,6 -1,6 +1,6 @@@
- --- config-2_1.yml 2022-10-25 11:32:05.718970156 +0100
- +++ config-2_1.yml.HIGHRES 2022-10-25 11:32:31.360769127 +0100
- @@ -149,14 +149,14 @@
---- config-2_1.yml 2022-11-15 12:30:50.000000000 -0500
-+++ config-2_1.yml.HIGHRES 2022-11-15 12:35:46.000000000 -0500
-@@ -157,14 +157,14 @@
++--- config-2_1.yml 2022-11-15 12:54:54.000000000 -0500
+++++ config-2_1.yml.HIGHRES 2022-11-15 12:56:26.000000000 -0500
++@@ -158,14 +158,14 @@
j8_par_executor: &j8_par_executor
executor:
name: java8-executor
@@@ -19,7 -19,7 +19,7 @@@
j8_small_executor: &j8_small_executor
executor:
- @@ -167,68 +167,74 @@
-@@ -175,74 +175,80 @@
++@@ -176,26 +176,26 @@
j8_medium_par_executor: &j8_medium_par_executor
executor:
name: java8-executor
@@@ -48,17 -47,24 +47,18 @@@
executor:
name: java11-executor
- #exec_resource_class: xlarge
- + exec_resource_class: medium
- parallelism: 1
+ - parallelism: 1
+ + exec_resource_class: xlarge
+ + parallelism: 5
- -j11_small_executor: &j11_small_executor
- +j11_small_par_executor: &j11_small_par_executor
+ j11_small_executor: &j11_small_executor
+ executor:
- name: java11-executor
-- #exec_resource_class: medium
-+ exec_resource_class: medium
- parallelism: 1
-
- j11_medium_par_executor: &j11_medium_par_executor
++@@ -207,43 +207,49 @@
executor:
name: java11-executor
- - #exec_resource_class: medium
-- #exec_resource_class: xlarge
++ #exec_resource_class: xlarge
- parallelism: 1
--+ exec_resource_class: xlarge
- + parallelism: 5
+ + parallelism: 2
j11_seq_executor: &j11_seq_executor
executor:
@@@ -111,3 -117,39 +111,39 @@@
j8_separate_jobs: &j8_separate_jobs
jobs:
-@@ -2070,7 +2076,7 @@
++@@ -2155,7 +2161,7 @@
+ python_version: '3.8'
+
+ j8_dtests_large_vnode:
+ - <<: *j8_par_executor
+ + <<: *j8_seq_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
-@@ -2084,7 +2090,7 @@
++@@ -2169,7 +2175,7 @@
+ pytest_extra_args: '--use-vnodes --num-tokens=16 --only-resource-intensive-tests --force-resource-intensive-tests'
+
+ j8_dtests_large:
+ - <<: *j8_par_executor
+ + <<: *j8_seq_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
-@@ -2098,7 +2104,7 @@
++@@ -2183,7 +2189,7 @@
+ pytest_extra_args: '--only-resource-intensive-tests --force-resource-intensive-tests'
+
+ j11_dtests_large_vnode:
+ - <<: *j11_par_executor
+ + <<: *j11_seq_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
-@@ -2112,7 +2118,7 @@
++@@ -2197,7 +2203,7 @@
+ pytest_extra_args: '--use-vnodes --num-tokens=16 --only-resource-intensive-tests --force-resource-intensive-tests'
+
+ j11_dtests_large:
+ - <<: *j11_par_executor
+ + <<: *j11_seq_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
diff --cc .circleci/config-2_1.yml.mid_res.patch
index 5e48831a17,3325f39485..223200c324
--- a/.circleci/config-2_1.yml.mid_res.patch
+++ b/.circleci/config-2_1.yml.mid_res.patch
@@@ -1,6 -1,6 +1,6 @@@
- --- config-2_1.yml 2022-11-09 14:47:20.000000000 -0500
- +++ config-2_1.yml.MIDRES 2022-11-09 14:55:10.000000000 -0500
- @@ -149,14 +149,14 @@
---- config-2_1.yml 2022-11-15 12:30:50.000000000 -0500
-+++ config-2_1.yml.MIDRES 2022-11-15 12:35:46.000000000 -0500
-@@ -157,14 +157,14 @@
++--- config-2_1.yml 2022-11-15 12:54:54.000000000 -0500
+++++ config-2_1.yml.MIDRES 2022-11-15 12:56:26.000000000 -0500
++@@ -158,14 +158,14 @@
j8_par_executor: &j8_par_executor
executor:
name: java8-executor
@@@ -19,7 -19,7 +19,7 @@@
j8_small_executor: &j8_small_executor
executor:
- @@ -164,29 +164,41 @@
-@@ -172,29 +172,41 @@
++@@ -173,29 +173,41 @@
exec_resource_class: medium
parallelism: 1
@@@ -68,10 -68,15 +68,15 @@@
j11_small_executor: &j11_small_executor
executor:
- @@ -194,41 +206,53 @@
- #exec_resource_class: medium
- parallelism: 1
-
-@@ -205,44 +217,56 @@
++@@ -206,44 +218,56 @@
+ j11_medium_par_executor: &j11_medium_par_executor
+ executor:
+ name: java11-executor
+ - #exec_resource_class: xlarge
+ - parallelism: 1
+ + exec_resource_class: large
+ + parallelism: 4
+ +
+j11_large_par_executor: &j11_large_par_executor
+ executor:
+ name: java11-executor
@@@ -129,7 -134,7 +134,7 @@@
j8_separate_jobs: &j8_separate_jobs
jobs:
- @@ -1612,7 +1636,7 @@
-@@ -1771,7 +1795,7 @@
++@@ -1856,7 +1880,7 @@
target: testclasslist-system-keyspace-directory
j8_dtests_vnode:
@@@ -138,7 -143,7 +143,7 @@@
steps:
- attach_workspace:
at: /home/cassandra
- @@ -1626,7 +1650,7 @@
-@@ -1785,7 +1809,7 @@
++@@ -1870,7 +1894,7 @@
pytest_extra_args: '--use-vnodes --num-tokens=16 --skip-resource-intensive-tests'
j8_dtests_offheap:
@@@ -147,7 -152,7 +152,7 @@@
steps:
- attach_workspace:
at: /home/cassandra
- @@ -1640,7 +1664,7 @@
-@@ -1799,7 +1823,7 @@
++@@ -1884,7 +1908,7 @@
pytest_extra_args: '--use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests'
j11_dtests_vnode:
@@@ -156,7 -161,7 +161,7 @@@
steps:
- attach_workspace:
at: /home/cassandra
- @@ -1655,7 +1679,7 @@
-@@ -1814,7 +1838,7 @@
++@@ -1899,7 +1923,7 @@
pytest_extra_args: '--use-vnodes --num-tokens=16 --skip-resource-intensive-tests'
j11_dtests_offheap:
@@@ -165,7 -170,7 +170,7 @@@
steps:
- attach_workspace:
at: /home/cassandra
- @@ -1670,7 +1694,7 @@
-@@ -1829,7 +1853,7 @@
++@@ -1914,7 +1938,7 @@
pytest_extra_args: '--use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests'
j8_dtests:
@@@ -174,7 -179,7 +179,7 @@@
steps:
- attach_workspace:
at: /home/cassandra
- @@ -1684,7 +1708,7 @@
-@@ -1843,7 +1867,7 @@
++@@ -1928,7 +1952,7 @@
pytest_extra_args: '--skip-resource-intensive-tests'
j11_dtests:
@@@ -183,7 -188,7 +188,7 @@@
steps:
- attach_workspace:
at: /home/cassandra
- @@ -1699,7 +1723,7 @@
-@@ -1858,7 +1882,7 @@
++@@ -1943,7 +1967,7 @@
pytest_extra_args: '--skip-resource-intensive-tests'
j8_upgrade_dtests:
@@@ -192,7 -197,7 +197,7 @@@
steps:
- attach_workspace:
at: /home/cassandra
- @@ -1713,7 +1737,7 @@
-@@ -1872,7 +1896,7 @@
++@@ -1957,7 +1981,7 @@
pytest_extra_args: '--execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all'
j8_cqlsh_dtests_py3_vnode:
@@@ -201,7 -206,7 +206,7 @@@
steps:
- attach_workspace:
at: /home/cassandra
- @@ -1728,7 +1752,7 @@
-@@ -1887,7 +1911,7 @@
++@@ -1972,7 +1996,7 @@
extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
j8_cqlsh_dtests_py3_offheap:
@@@ -210,7 -215,7 +215,7 @@@
steps:
- attach_workspace:
at: /home/cassandra
- @@ -1743,7 +1767,7 @@
-@@ -1902,7 +1926,7 @@
++@@ -1987,7 +2011,7 @@
extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
j8_cqlsh_dtests_py38_vnode:
@@@ -219,7 -224,7 +224,7 @@@
steps:
- attach_workspace:
at: /home/cassandra
- @@ -1761,7 +1785,7 @@
-@@ -1920,7 +1944,7 @@
++@@ -2005,7 +2029,7 @@
python_version: '3.8'
j8_cqlsh_dtests_py38_offheap:
@@@ -228,7 -233,7 +233,7 @@@
steps:
- attach_workspace:
at: /home/cassandra
- @@ -1779,7 +1803,7 @@
-@@ -1938,7 +1962,7 @@
++@@ -2023,7 +2047,7 @@
python_version: '3.8'
j8_cqlsh_dtests_py3:
@@@ -237,7 -242,7 +242,7 @@@
steps:
- attach_workspace:
at: /home/cassandra
- @@ -1794,7 +1818,7 @@
-@@ -1953,7 +1977,7 @@
++@@ -2038,7 +2062,7 @@
extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
j8_cqlsh_dtests_py38:
@@@ -246,7 -251,7 +251,7 @@@
steps:
- attach_workspace:
at: /home/cassandra
- @@ -1812,7 +1836,7 @@
-@@ -1971,7 +1995,7 @@
++@@ -2056,7 +2080,7 @@
python_version: '3.8'
j11_cqlsh_dtests_py3_vnode:
@@@ -255,7 -260,7 +260,7 @@@
steps:
- attach_workspace:
at: /home/cassandra
- @@ -1827,7 +1851,7 @@
-@@ -1986,7 +2010,7 @@
++@@ -2071,7 +2095,7 @@
extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
j11_cqlsh_dtests_py3_offheap:
@@@ -264,7 -269,7 +269,7 @@@
steps:
- attach_workspace:
at: /home/cassandra
- @@ -1842,7 +1866,7 @@
-@@ -2001,7 +2025,7 @@
++@@ -2086,7 +2110,7 @@
extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
j11_cqlsh_dtests_py38_vnode:
@@@ -273,7 -278,7 +278,7 @@@
steps:
- attach_workspace:
at: /home/cassandra
- @@ -1860,7 +1884,7 @@
-@@ -2019,7 +2043,7 @@
++@@ -2104,7 +2128,7 @@
python_version: '3.8'
j11_cqlsh_dtests_py38_offheap:
@@@ -282,7 -287,7 +287,7 @@@
steps:
- attach_workspace:
at: /home/cassandra
- @@ -1878,7 +1902,7 @@
-@@ -2037,7 +2061,7 @@
++@@ -2122,7 +2146,7 @@
python_version: '3.8'
j11_cqlsh_dtests_py3:
@@@ -291,7 -296,7 +296,7 @@@
steps:
- attach_workspace:
at: /home/cassandra
- @@ -1893,7 +1917,7 @@
-@@ -2052,7 +2076,7 @@
++@@ -2137,7 +2161,7 @@
extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
j11_cqlsh_dtests_py38:
@@@ -300,3 -305,39 +305,39 @@@
steps:
- attach_workspace:
at: /home/cassandra
-@@ -2070,7 +2094,7 @@
++@@ -2155,7 +2179,7 @@
+ python_version: '3.8'
+
+ j8_dtests_large_vnode:
+ - <<: *j8_par_executor
+ + <<: *j8_medium_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
-@@ -2084,7 +2108,7 @@
++@@ -2169,7 +2193,7 @@
+ pytest_extra_args: '--use-vnodes --num-tokens=16 --only-resource-intensive-tests --force-resource-intensive-tests'
+
+ j8_dtests_large:
+ - <<: *j8_par_executor
+ + <<: *j8_medium_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
-@@ -2098,7 +2122,7 @@
++@@ -2183,7 +2207,7 @@
+ pytest_extra_args: '--only-resource-intensive-tests --force-resource-intensive-tests'
+
+ j11_dtests_large_vnode:
+ - <<: *j11_par_executor
+ + <<: *j11_medium_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
-@@ -2112,7 +2136,7 @@
++@@ -2197,7 +2221,7 @@
+ pytest_extra_args: '--use-vnodes --num-tokens=16 --only-resource-intensive-tests --force-resource-intensive-tests'
+
+ j11_dtests_large:
+ - <<: *j11_par_executor
+ + <<: *j11_medium_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
diff --cc .circleci/config.yml
index 737c361a83,323567f960..dfc112a96f
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@@ -659,95 -823,6 +823,97 @@@ jobs
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- CASSANDRA_USE_JDK11: true
+ j8_utests_trie_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically d [...]
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
++ - REPEATED_LARGE_DTESTS: null
++ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_jvm_dtests_vnode:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
@@@ -1679,36 -1760,369 +1852,369 @@@
- attach_workspace:
at: /home/cassandra
- run:
- name: Log Environment Information
+ name: Clone Cassandra dtest Repository (via git)
command: |
- echo '*** id ***'
- id
- echo '*** cat /proc/cpuinfo ***'
- cat /proc/cpuinfo
- echo '*** free -m ***'
- free -m
- echo '*** df -m ***'
- df -m
- echo '*** ifconfig -a ***'
- ifconfig -a
- echo '*** uname -a ***'
- uname -a
- echo '*** mount ***'
- mount
- echo '*** env ***'
- env
- echo '*** java ***'
- which java
- java -version
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- run:
- name: Repeatedly run new or modifed JUnit tests
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Run repeated Python DTests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically d [...]
+ command: |
+ if [ "${REPEATED_LARGE_DTESTS}" == "<nil>" ]; then
+ echo "Repeated dtest name hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_LARGE_DTESTS_COUNT}" == "<nil>" ]; then
+ echo "Repeated dtest count hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_LARGE_DTESTS_COUNT}" -le 0 ]; then
+ echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
+ else
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((${REPEATED_LARGE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_LARGE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running ${REPEATED_LARGE_DTESTS} $count times"
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+
+ tests_arg=$(echo ${REPEATED_LARGE_DTESTS} | sed -e "s/,/ /g")
+
+ stop_on_failure_arg=""
+ if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
+ stop_on_failure_arg="-x"
+ fi
+
+ vnodes_args=""
+ if true; then
+ vnodes_args="--use-vnodes --num-tokens=16"
+ fi
+
+ upgrade_arg=""
+ if false; then
+ upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
+ fi
+
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir --only-resource-intensive-tests --force-resource-intensive-tests $tests_arg | tee /tmp/dtest/stdout.txt
+ fi
+ fi
- store_test_results:
- path: /tmp/results/repeated_utests/output
+ path: /tmp/results
- store_artifacts:
- path: /tmp/results/repeated_utests/stdout
- destination: stdout
+ path: /tmp/dtest
+ destination: dtest
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_cqlsh_dtests_py38_offheap:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_dtests_offheap)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_dtests_offheap)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --u [...]
+ - run:
+ name: Run dtests (j11_dtests_offheap)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt"
+ cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt
+
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.8
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt`
+ if [ ! -z "$SPLIT_TESTS" ]; then
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j11_dtests_offheap.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ else
+ echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
+ (exit 1)
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_dtests_offheap
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_dtests_offheap_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_dtests_large:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_large_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_large_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --only-res [...]
+ - run:
+ name: Run dtests (j11_large_without_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j11_large_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j11_large_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from [...]
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_large_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_large_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_utests_system_keyspace_directory_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically d [...]
++ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically d [...]
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
- store_artifacts:
path: /tmp/results/repeated_utests/output
destination: junitxml
@@@ -4279,81 -5053,96 +5145,216 @@@
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j11_utests_trie:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist-trie)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
- ant testclasslist-trie -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
- no_output_timeout: 15m
++ ant testclasslist-trie -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
++ no_output_timeout: 15m
++ - store_test_results:
++ path: /tmp/cassandra/build/test/output/
++ - store_artifacts:
++ path: /tmp/cassandra/build/test/output
++ destination: junitxml
++ - store_artifacts:
++ path: /tmp/cassandra/build/test/logs
++ destination: logs
++ environment:
++ - ANT_HOME: /usr/share/ant
++ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ - LANG: en_US.UTF-8
++ - KEEP_TEST_DIR: true
++ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
++ - PYTHONIOENCODING: utf-8
++ - PYTHONUNBUFFERED: true
++ - CASS_DRIVER_NO_EXTENSIONS: true
++ - CASS_DRIVER_NO_CYTHON: true
++ - CASSANDRA_SKIP_SYNC: true
++ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
++ - DTEST_BRANCH: trunk
++ - CCM_MAX_HEAP_SIZE: 1024M
++ - CCM_HEAP_NEWSIZE: 256M
++ - REPEATED_TESTS_STOP_ON_FAILURE: false
++ - REPEATED_UTESTS: null
++ - REPEATED_UTESTS_COUNT: 500
++ - REPEATED_UTESTS_FQLTOOL: null
++ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
++ - REPEATED_UTESTS_LONG: null
++ - REPEATED_UTESTS_LONG_COUNT: 100
++ - REPEATED_UTESTS_STRESS: null
++ - REPEATED_UTESTS_STRESS_COUNT: 500
++ - REPEATED_SIMULATOR_DTESTS: null
++ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
++ - REPEATED_JVM_DTESTS: null
++ - REPEATED_JVM_DTESTS_COUNT: 500
++ - REPEATED_JVM_UPGRADE_DTESTS: null
++ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
++ - REPEATED_DTESTS: null
++ - REPEATED_DTESTS_COUNT: 500
++ - REPEATED_LARGE_DTESTS: null
++ - REPEATED_LARGE_DTESTS_COUNT: 100
++ - REPEATED_UPGRADE_DTESTS: null
++ - REPEATED_UPGRADE_DTESTS_COUNT: 25
++ - REPEATED_ANT_TEST_TARGET: testsome
++ - REPEATED_ANT_TEST_CLASS: null
++ - REPEATED_ANT_TEST_METHODS: null
++ - REPEATED_ANT_TEST_VNODES: false
++ - REPEATED_ANT_TEST_COUNT: 500
++ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - CASSANDRA_USE_JDK11: true
+ j11_dtests_large_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Run repeated Python DTests
+ no_output_timeout: 15m
+ command: |
+ if [ "${REPEATED_LARGE_DTESTS}" == "<nil>" ]; then
+ echo "Repeated dtest name hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_LARGE_DTESTS_COUNT}" == "<nil>" ]; then
+ echo "Repeated dtest count hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_LARGE_DTESTS_COUNT}" -le 0 ]; then
+ echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
+ else
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((${REPEATED_LARGE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_LARGE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running ${REPEATED_LARGE_DTESTS} $count times"
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+
+ tests_arg=$(echo ${REPEATED_LARGE_DTESTS} | sed -e "s/,/ /g")
+
+ stop_on_failure_arg=""
+ if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
+ stop_on_failure_arg="-x"
+ fi
+
+ vnodes_args=""
+ if false; then
+ vnodes_args="--use-vnodes --num-tokens=16"
+ fi
+
+ upgrade_arg=""
+ if false; then
+ upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
+ fi
+
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir --only-resource-intensive-tests --force-resource-intensive-tests $tests_arg | tee /tmp/dtest/stdout.txt
+ fi
+ fi
- store_test_results:
- path: /tmp/cassandra/build/test/output/
+ path: /tmp/results
- store_artifacts:
- path: /tmp/cassandra/build/test/output
- destination: junitxml
+ path: /tmp/dtest
+ destination: dtest
- store_artifacts:
- path: /tmp/cassandra/build/test/logs
- destination: logs
+ path: ~/cassandra-dtest/logs
+ destination: dtest_logs
environment:
- ANT_HOME: /usr/share/ant
- JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
@@@ -4397,123 -5188,6 +5400,125 @@@
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- CASSANDRA_USE_JDK11: true
+ j8_utests_trie:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist-trie)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant testclasslist-trie -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
++ - REPEATED_LARGE_DTESTS: null
++ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_cqlsh_dtests_py3_offheap:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
@@@ -5326,96 -6031,6 +6362,98 @@@
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- CASSANDRA_USE_JDK11: true
+ j11_utests_trie_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically d [...]
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
++ - REPEATED_LARGE_DTESTS: null
++ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
j11_jvm_dtests_repeat:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11:latest
diff --cc .circleci/config.yml.HIGHRES
index cfbf494949,219586716f..c85764361d
--- a/.circleci/config.yml.HIGHRES
+++ b/.circleci/config.yml.HIGHRES
@@@ -659,95 -823,6 +823,97 @@@ jobs
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- CASSANDRA_USE_JDK11: true
+ j8_utests_trie_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically d [...]
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
++ - REPEATED_LARGE_DTESTS: null
++ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_jvm_dtests_vnode:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
@@@ -1679,36 -1760,369 +1852,369 @@@
- attach_workspace:
at: /home/cassandra
- run:
- name: Log Environment Information
+ name: Clone Cassandra dtest Repository (via git)
command: |
- echo '*** id ***'
- id
- echo '*** cat /proc/cpuinfo ***'
- cat /proc/cpuinfo
- echo '*** free -m ***'
- free -m
- echo '*** df -m ***'
- df -m
- echo '*** ifconfig -a ***'
- ifconfig -a
- echo '*** uname -a ***'
- uname -a
- echo '*** mount ***'
- mount
- echo '*** env ***'
- env
- echo '*** java ***'
- which java
- java -version
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- run:
- name: Repeatedly run new or modifed JUnit tests
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Run repeated Python DTests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically d [...]
+ command: |
+ if [ "${REPEATED_LARGE_DTESTS}" == "<nil>" ]; then
+ echo "Repeated dtest name hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_LARGE_DTESTS_COUNT}" == "<nil>" ]; then
+ echo "Repeated dtest count hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_LARGE_DTESTS_COUNT}" -le 0 ]; then
+ echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
+ else
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((${REPEATED_LARGE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_LARGE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running ${REPEATED_LARGE_DTESTS} $count times"
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+
+ tests_arg=$(echo ${REPEATED_LARGE_DTESTS} | sed -e "s/,/ /g")
+
+ stop_on_failure_arg=""
+ if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
+ stop_on_failure_arg="-x"
+ fi
+
+ vnodes_args=""
+ if true; then
+ vnodes_args="--use-vnodes --num-tokens=16"
+ fi
+
+ upgrade_arg=""
+ if false; then
+ upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
+ fi
+
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir --only-resource-intensive-tests --force-resource-intensive-tests $tests_arg | tee /tmp/dtest/stdout.txt
+ fi
+ fi
- store_test_results:
- path: /tmp/results/repeated_utests/output
+ path: /tmp/results
- store_artifacts:
- path: /tmp/results/repeated_utests/stdout
- destination: stdout
+ path: /tmp/dtest
+ destination: dtest
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_cqlsh_dtests_py38_offheap:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_dtests_offheap)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_dtests_offheap)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --u [...]
+ - run:
+ name: Run dtests (j11_dtests_offheap)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt"
+ cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt
+
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.8
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt`
+ if [ ! -z "$SPLIT_TESTS" ]; then
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j11_dtests_offheap.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ else
+ echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
+ (exit 1)
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_dtests_offheap
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_dtests_offheap_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_dtests_large:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_large_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_large_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --only-res [...]
+ - run:
+ name: Run dtests (j11_large_without_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j11_large_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j11_large_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from [...]
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_large_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_large_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_utests_system_keyspace_directory_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically d [...]
++ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically d [...]
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
- store_artifacts:
path: /tmp/results/repeated_utests/output
destination: junitxml
@@@ -4279,81 -5053,96 +5145,216 @@@
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j11_utests_trie:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist-trie)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
- ant testclasslist-trie -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
- no_output_timeout: 15m
++ ant testclasslist-trie -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
++ no_output_timeout: 15m
++ - store_test_results:
++ path: /tmp/cassandra/build/test/output/
++ - store_artifacts:
++ path: /tmp/cassandra/build/test/output
++ destination: junitxml
++ - store_artifacts:
++ path: /tmp/cassandra/build/test/logs
++ destination: logs
++ environment:
++ - ANT_HOME: /usr/share/ant
++ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ - LANG: en_US.UTF-8
++ - KEEP_TEST_DIR: true
++ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
++ - PYTHONIOENCODING: utf-8
++ - PYTHONUNBUFFERED: true
++ - CASS_DRIVER_NO_EXTENSIONS: true
++ - CASS_DRIVER_NO_CYTHON: true
++ - CASSANDRA_SKIP_SYNC: true
++ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
++ - DTEST_BRANCH: trunk
++ - CCM_MAX_HEAP_SIZE: 1024M
++ - CCM_HEAP_NEWSIZE: 256M
++ - REPEATED_TESTS_STOP_ON_FAILURE: false
++ - REPEATED_UTESTS: null
++ - REPEATED_UTESTS_COUNT: 500
++ - REPEATED_UTESTS_FQLTOOL: null
++ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
++ - REPEATED_UTESTS_LONG: null
++ - REPEATED_UTESTS_LONG_COUNT: 100
++ - REPEATED_UTESTS_STRESS: null
++ - REPEATED_UTESTS_STRESS_COUNT: 500
++ - REPEATED_SIMULATOR_DTESTS: null
++ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
++ - REPEATED_JVM_DTESTS: null
++ - REPEATED_JVM_DTESTS_COUNT: 500
++ - REPEATED_JVM_UPGRADE_DTESTS: null
++ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
++ - REPEATED_DTESTS: null
++ - REPEATED_DTESTS_COUNT: 500
++ - REPEATED_LARGE_DTESTS: null
++ - REPEATED_LARGE_DTESTS_COUNT: 100
++ - REPEATED_UPGRADE_DTESTS: null
++ - REPEATED_UPGRADE_DTESTS_COUNT: 25
++ - REPEATED_ANT_TEST_TARGET: testsome
++ - REPEATED_ANT_TEST_CLASS: null
++ - REPEATED_ANT_TEST_METHODS: null
++ - REPEATED_ANT_TEST_VNODES: false
++ - REPEATED_ANT_TEST_COUNT: 500
++ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - CASSANDRA_USE_JDK11: true
+ j11_dtests_large_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Run repeated Python DTests
+ no_output_timeout: 15m
+ command: |
+ if [ "${REPEATED_LARGE_DTESTS}" == "<nil>" ]; then
+ echo "Repeated dtest name hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_LARGE_DTESTS_COUNT}" == "<nil>" ]; then
+ echo "Repeated dtest count hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_LARGE_DTESTS_COUNT}" -le 0 ]; then
+ echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
+ else
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((${REPEATED_LARGE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_LARGE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running ${REPEATED_LARGE_DTESTS} $count times"
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+
+ tests_arg=$(echo ${REPEATED_LARGE_DTESTS} | sed -e "s/,/ /g")
+
+ stop_on_failure_arg=""
+ if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
+ stop_on_failure_arg="-x"
+ fi
+
+ vnodes_args=""
+ if false; then
+ vnodes_args="--use-vnodes --num-tokens=16"
+ fi
+
+ upgrade_arg=""
+ if false; then
+ upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
+ fi
+
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir --only-resource-intensive-tests --force-resource-intensive-tests $tests_arg | tee /tmp/dtest/stdout.txt
+ fi
+ fi
- store_test_results:
- path: /tmp/cassandra/build/test/output/
+ path: /tmp/results
- store_artifacts:
- path: /tmp/cassandra/build/test/output
- destination: junitxml
+ path: /tmp/dtest
+ destination: dtest
- store_artifacts:
- path: /tmp/cassandra/build/test/logs
- destination: logs
+ path: ~/cassandra-dtest/logs
+ destination: dtest_logs
environment:
- ANT_HOME: /usr/share/ant
- JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
@@@ -4397,123 -5188,6 +5400,125 @@@
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- CASSANDRA_USE_JDK11: true
+ j8_utests_trie:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist-trie)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant testclasslist-trie -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
++ - REPEATED_LARGE_DTESTS: null
++ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_cqlsh_dtests_py3_offheap:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
@@@ -5326,96 -6031,6 +6362,98 @@@
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- CASSANDRA_USE_JDK11: true
+ j11_utests_trie_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically d [...]
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
++ - REPEATED_LARGE_DTESTS: null
++ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
j11_jvm_dtests_repeat:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11:latest
diff --cc .circleci/config.yml.LOWRES
index 737c361a83,323567f960..dfc112a96f
--- a/.circleci/config.yml.LOWRES
+++ b/.circleci/config.yml.LOWRES
@@@ -659,95 -823,6 +823,97 @@@ jobs
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- CASSANDRA_USE_JDK11: true
+ j8_utests_trie_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically d [...]
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
++ - REPEATED_LARGE_DTESTS: null
++ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_jvm_dtests_vnode:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
@@@ -1679,36 -1760,369 +1852,369 @@@
- attach_workspace:
at: /home/cassandra
- run:
- name: Log Environment Information
+ name: Clone Cassandra dtest Repository (via git)
command: |
- echo '*** id ***'
- id
- echo '*** cat /proc/cpuinfo ***'
- cat /proc/cpuinfo
- echo '*** free -m ***'
- free -m
- echo '*** df -m ***'
- df -m
- echo '*** ifconfig -a ***'
- ifconfig -a
- echo '*** uname -a ***'
- uname -a
- echo '*** mount ***'
- mount
- echo '*** env ***'
- env
- echo '*** java ***'
- which java
- java -version
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- run:
- name: Repeatedly run new or modifed JUnit tests
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Run repeated Python DTests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically d [...]
+ command: |
+ if [ "${REPEATED_LARGE_DTESTS}" == "<nil>" ]; then
+ echo "Repeated dtest name hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_LARGE_DTESTS_COUNT}" == "<nil>" ]; then
+ echo "Repeated dtest count hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_LARGE_DTESTS_COUNT}" -le 0 ]; then
+ echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
+ else
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((${REPEATED_LARGE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_LARGE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running ${REPEATED_LARGE_DTESTS} $count times"
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+
+ tests_arg=$(echo ${REPEATED_LARGE_DTESTS} | sed -e "s/,/ /g")
+
+ stop_on_failure_arg=""
+ if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
+ stop_on_failure_arg="-x"
+ fi
+
+ vnodes_args=""
+ if true; then
+ vnodes_args="--use-vnodes --num-tokens=16"
+ fi
+
+ upgrade_arg=""
+ if false; then
+ upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
+ fi
+
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir --only-resource-intensive-tests --force-resource-intensive-tests $tests_arg | tee /tmp/dtest/stdout.txt
+ fi
+ fi
- store_test_results:
- path: /tmp/results/repeated_utests/output
+ path: /tmp/results
- store_artifacts:
- path: /tmp/results/repeated_utests/stdout
- destination: stdout
+ path: /tmp/dtest
+ destination: dtest
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_cqlsh_dtests_py38_offheap:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_dtests_offheap)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_dtests_offheap)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --u [...]
+ - run:
+ name: Run dtests (j11_dtests_offheap)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt"
+ cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt
+
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.8
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt`
+ if [ ! -z "$SPLIT_TESTS" ]; then
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j11_dtests_offheap.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ else
+ echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
+ (exit 1)
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_dtests_offheap
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_dtests_offheap_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_dtests_large:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_large_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_large_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --only-res [...]
+ - run:
+ name: Run dtests (j11_large_without_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j11_large_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j11_large_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from [...]
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_large_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_large_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_utests_system_keyspace_directory_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically d [...]
++ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically d [...]
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
- store_artifacts:
path: /tmp/results/repeated_utests/output
destination: junitxml
@@@ -4279,81 -5053,96 +5145,216 @@@
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j11_utests_trie:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist-trie)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
- ant testclasslist-trie -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
- no_output_timeout: 15m
++ ant testclasslist-trie -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
++ no_output_timeout: 15m
++ - store_test_results:
++ path: /tmp/cassandra/build/test/output/
++ - store_artifacts:
++ path: /tmp/cassandra/build/test/output
++ destination: junitxml
++ - store_artifacts:
++ path: /tmp/cassandra/build/test/logs
++ destination: logs
++ environment:
++ - ANT_HOME: /usr/share/ant
++ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ - LANG: en_US.UTF-8
++ - KEEP_TEST_DIR: true
++ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
++ - PYTHONIOENCODING: utf-8
++ - PYTHONUNBUFFERED: true
++ - CASS_DRIVER_NO_EXTENSIONS: true
++ - CASS_DRIVER_NO_CYTHON: true
++ - CASSANDRA_SKIP_SYNC: true
++ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
++ - DTEST_BRANCH: trunk
++ - CCM_MAX_HEAP_SIZE: 1024M
++ - CCM_HEAP_NEWSIZE: 256M
++ - REPEATED_TESTS_STOP_ON_FAILURE: false
++ - REPEATED_UTESTS: null
++ - REPEATED_UTESTS_COUNT: 500
++ - REPEATED_UTESTS_FQLTOOL: null
++ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
++ - REPEATED_UTESTS_LONG: null
++ - REPEATED_UTESTS_LONG_COUNT: 100
++ - REPEATED_UTESTS_STRESS: null
++ - REPEATED_UTESTS_STRESS_COUNT: 500
++ - REPEATED_SIMULATOR_DTESTS: null
++ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
++ - REPEATED_JVM_DTESTS: null
++ - REPEATED_JVM_DTESTS_COUNT: 500
++ - REPEATED_JVM_UPGRADE_DTESTS: null
++ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
++ - REPEATED_DTESTS: null
++ - REPEATED_DTESTS_COUNT: 500
++ - REPEATED_LARGE_DTESTS: null
++ - REPEATED_LARGE_DTESTS_COUNT: 100
++ - REPEATED_UPGRADE_DTESTS: null
++ - REPEATED_UPGRADE_DTESTS_COUNT: 25
++ - REPEATED_ANT_TEST_TARGET: testsome
++ - REPEATED_ANT_TEST_CLASS: null
++ - REPEATED_ANT_TEST_METHODS: null
++ - REPEATED_ANT_TEST_VNODES: false
++ - REPEATED_ANT_TEST_COUNT: 500
++ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - CASSANDRA_USE_JDK11: true
+ j11_dtests_large_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Run repeated Python DTests
+ no_output_timeout: 15m
+ command: |
+ if [ "${REPEATED_LARGE_DTESTS}" == "<nil>" ]; then
+ echo "Repeated dtest name hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_LARGE_DTESTS_COUNT}" == "<nil>" ]; then
+ echo "Repeated dtest count hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_LARGE_DTESTS_COUNT}" -le 0 ]; then
+ echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
+ else
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((${REPEATED_LARGE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_LARGE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running ${REPEATED_LARGE_DTESTS} $count times"
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+
+ tests_arg=$(echo ${REPEATED_LARGE_DTESTS} | sed -e "s/,/ /g")
+
+ stop_on_failure_arg=""
+ if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
+ stop_on_failure_arg="-x"
+ fi
+
+ vnodes_args=""
+ if false; then
+ vnodes_args="--use-vnodes --num-tokens=16"
+ fi
+
+ upgrade_arg=""
+ if false; then
+ upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
+ fi
+
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir --only-resource-intensive-tests --force-resource-intensive-tests $tests_arg | tee /tmp/dtest/stdout.txt
+ fi
+ fi
- store_test_results:
- path: /tmp/cassandra/build/test/output/
+ path: /tmp/results
- store_artifacts:
- path: /tmp/cassandra/build/test/output
- destination: junitxml
+ path: /tmp/dtest
+ destination: dtest
- store_artifacts:
- path: /tmp/cassandra/build/test/logs
- destination: logs
+ path: ~/cassandra-dtest/logs
+ destination: dtest_logs
environment:
- ANT_HOME: /usr/share/ant
- JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
@@@ -4397,123 -5188,6 +5400,125 @@@
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- CASSANDRA_USE_JDK11: true
+ j8_utests_trie:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist-trie)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant testclasslist-trie -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
++ - REPEATED_LARGE_DTESTS: null
++ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_cqlsh_dtests_py3_offheap:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
@@@ -5326,96 -6031,6 +6362,98 @@@
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- CASSANDRA_USE_JDK11: true
+ j11_utests_trie_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically d [...]
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
++ - REPEATED_LARGE_DTESTS: null
++ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
j11_jvm_dtests_repeat:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11:latest
diff --cc .circleci/config.yml.MIDRES
index 812636db39,f6fae8753e..87f0f9fb29
--- a/.circleci/config.yml.MIDRES
+++ b/.circleci/config.yml.MIDRES
@@@ -659,95 -823,6 +823,97 @@@ jobs
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- CASSANDRA_USE_JDK11: true
+ j8_utests_trie_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 25
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically d [...]
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
++ - REPEATED_LARGE_DTESTS: null
++ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_jvm_dtests_vnode:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
@@@ -1679,33 -1760,366 +1852,366 @@@
- attach_workspace:
at: /home/cassandra
- run:
- name: Log Environment Information
+ name: Clone Cassandra dtest Repository (via git)
command: |
- echo '*** id ***'
- id
- echo '*** cat /proc/cpuinfo ***'
- cat /proc/cpuinfo
- echo '*** free -m ***'
- free -m
- echo '*** df -m ***'
- df -m
- echo '*** ifconfig -a ***'
- ifconfig -a
- echo '*** uname -a ***'
- uname -a
- echo '*** mount ***'
- mount
- echo '*** env ***'
- env
- echo '*** java ***'
- which java
- java -version
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- run:
- name: Repeatedly run new or modifed JUnit tests
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Run repeated Python DTests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically d [...]
+ command: |
+ if [ "${REPEATED_LARGE_DTESTS}" == "<nil>" ]; then
+ echo "Repeated dtest name hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_LARGE_DTESTS_COUNT}" == "<nil>" ]; then
+ echo "Repeated dtest count hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_LARGE_DTESTS_COUNT}" -le 0 ]; then
+ echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
+ else
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((${REPEATED_LARGE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_LARGE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running ${REPEATED_LARGE_DTESTS} $count times"
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+
+ tests_arg=$(echo ${REPEATED_LARGE_DTESTS} | sed -e "s/,/ /g")
+
+ stop_on_failure_arg=""
+ if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
+ stop_on_failure_arg="-x"
+ fi
+
+ vnodes_args=""
+ if true; then
+ vnodes_args="--use-vnodes --num-tokens=16"
+ fi
+
+ upgrade_arg=""
+ if false; then
+ upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
+ fi
+
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir --only-resource-intensive-tests --force-resource-intensive-tests $tests_arg | tee /tmp/dtest/stdout.txt
+ fi
+ fi
- store_test_results:
- path: /tmp/results/repeated_utests/output
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_cqlsh_dtests_py38_offheap:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: large
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 50
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_dtests_offheap)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_dtests_offheap)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --u [...]
+ - run:
+ name: Run dtests (j11_dtests_offheap)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt"
+ cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt
+
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.8
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt`
+ if [ ! -z "$SPLIT_TESTS" ]; then
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j11_dtests_offheap.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ else
+ echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
+ (exit 1)
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_dtests_offheap
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_dtests_offheap_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_dtests_large:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: large
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_large_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_large_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --only-res [...]
+ - run:
+ name: Run dtests (j11_large_without_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j11_large_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j11_large_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from [...]
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_large_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_large_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_LARGE_DTESTS: null
+ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_utests_system_keyspace_directory_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 25
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically d [...]
++ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically d [...]
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
- store_artifacts:
path: /tmp/results/repeated_utests/stdout
destination: stdout
@@@ -4279,81 -5053,96 +5145,216 @@@
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j11_utests_trie:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 25
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist-trie)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
- ant testclasslist-trie -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
- no_output_timeout: 15m
++ ant testclasslist-trie -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
++ no_output_timeout: 15m
++ - store_test_results:
++ path: /tmp/cassandra/build/test/output/
++ - store_artifacts:
++ path: /tmp/cassandra/build/test/output
++ destination: junitxml
++ - store_artifacts:
++ path: /tmp/cassandra/build/test/logs
++ destination: logs
++ environment:
++ - ANT_HOME: /usr/share/ant
++ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ - LANG: en_US.UTF-8
++ - KEEP_TEST_DIR: true
++ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
++ - PYTHONIOENCODING: utf-8
++ - PYTHONUNBUFFERED: true
++ - CASS_DRIVER_NO_EXTENSIONS: true
++ - CASS_DRIVER_NO_CYTHON: true
++ - CASSANDRA_SKIP_SYNC: true
++ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
++ - DTEST_BRANCH: trunk
++ - CCM_MAX_HEAP_SIZE: 1024M
++ - CCM_HEAP_NEWSIZE: 256M
++ - REPEATED_TESTS_STOP_ON_FAILURE: false
++ - REPEATED_UTESTS: null
++ - REPEATED_UTESTS_COUNT: 500
++ - REPEATED_UTESTS_FQLTOOL: null
++ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
++ - REPEATED_UTESTS_LONG: null
++ - REPEATED_UTESTS_LONG_COUNT: 100
++ - REPEATED_UTESTS_STRESS: null
++ - REPEATED_UTESTS_STRESS_COUNT: 500
++ - REPEATED_SIMULATOR_DTESTS: null
++ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
++ - REPEATED_JVM_DTESTS: null
++ - REPEATED_JVM_DTESTS_COUNT: 500
++ - REPEATED_JVM_UPGRADE_DTESTS: null
++ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
++ - REPEATED_DTESTS: null
++ - REPEATED_DTESTS_COUNT: 500
++ - REPEATED_LARGE_DTESTS: null
++ - REPEATED_LARGE_DTESTS_COUNT: 100
++ - REPEATED_UPGRADE_DTESTS: null
++ - REPEATED_UPGRADE_DTESTS_COUNT: 25
++ - REPEATED_ANT_TEST_TARGET: testsome
++ - REPEATED_ANT_TEST_CLASS: null
++ - REPEATED_ANT_TEST_METHODS: null
++ - REPEATED_ANT_TEST_VNODES: false
++ - REPEATED_ANT_TEST_COUNT: 500
++ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - CASSANDRA_USE_JDK11: true
+ j11_dtests_large_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: large
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 25
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Run repeated Python DTests
+ no_output_timeout: 15m
+ command: |
+ if [ "${REPEATED_LARGE_DTESTS}" == "<nil>" ]; then
+ echo "Repeated dtest name hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_LARGE_DTESTS_COUNT}" == "<nil>" ]; then
+ echo "Repeated dtest count hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_LARGE_DTESTS_COUNT}" -le 0 ]; then
+ echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
+ else
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((${REPEATED_LARGE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_LARGE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running ${REPEATED_LARGE_DTESTS} $count times"
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+
+ tests_arg=$(echo ${REPEATED_LARGE_DTESTS} | sed -e "s/,/ /g")
+
+ stop_on_failure_arg=""
+ if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
+ stop_on_failure_arg="-x"
+ fi
+
+ vnodes_args=""
+ if false; then
+ vnodes_args="--use-vnodes --num-tokens=16"
+ fi
+
+ upgrade_arg=""
+ if false; then
+ upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
+ fi
+
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir --only-resource-intensive-tests --force-resource-intensive-tests $tests_arg | tee /tmp/dtest/stdout.txt
+ fi
+ fi
- store_test_results:
- path: /tmp/cassandra/build/test/output/
+ path: /tmp/results
- store_artifacts:
- path: /tmp/cassandra/build/test/output
- destination: junitxml
+ path: /tmp/dtest
+ destination: dtest
- store_artifacts:
- path: /tmp/cassandra/build/test/logs
- destination: logs
+ path: ~/cassandra-dtest/logs
+ destination: dtest_logs
environment:
- ANT_HOME: /usr/share/ant
- JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
@@@ -4397,123 -5188,6 +5400,125 @@@
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- CASSANDRA_USE_JDK11: true
+ j8_utests_trie:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 25
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist-trie)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant testclasslist-trie -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
++ - REPEATED_LARGE_DTESTS: null
++ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_cqlsh_dtests_py3_offheap:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
@@@ -5326,96 -6031,6 +6362,98 @@@
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- CASSANDRA_USE_JDK11: true
+ j11_utests_trie_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 25
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically d [...]
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
++ - REPEATED_LARGE_DTESTS: null
++ - REPEATED_LARGE_DTESTS_COUNT: 100
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
j11_jvm_dtests_repeat:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11:latest
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org