You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by ad...@apache.org on 2022/11/03 11:47:19 UTC
[cassandra] branch cassandra-4.0 updated (0db0b8f781 -> 6152cb08db)
This is an automated email from the ASF dual-hosted git repository.
adelapena pushed a change to branch cassandra-4.0
in repository https://gitbox.apache.org/repos/asf/cassandra.git
from 0db0b8f781 Update NetBeans project file for dependency changes since 7th July 2021
new 116ce3bc05 CircleCI: Add jobs for missing specialized unit tests
new 7df905a429 Merge branch 'cassandra-3.0' into cassandra-3.11
new 6152cb08db Merge branch 'cassandra-3.11' into cassandra-4.0
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.circleci/config-2_1.yml | 539 ++-
.circleci/config-2_1.yml.high_res.patch | 13 +-
.circleci/config-2_1.yml.mid_res.patch | 49 +-
.circleci/config.yml | 5913 +++++++++++++++++++++----------
.circleci/config.yml.HIGHRES | 5499 ++++++++++++++++++++--------
.circleci/config.yml.LOWRES | 5501 ++++++++++++++++++++--------
.circleci/config.yml.MIDRES | 5493 ++++++++++++++++++++--------
.circleci/generate.sh | 17 +-
8 files changed, 16443 insertions(+), 6581 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org
[cassandra] 01/01: Merge branch 'cassandra-3.11' into cassandra-4.0
Posted by ad...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
adelapena pushed a commit to branch cassandra-4.0
in repository https://gitbox.apache.org/repos/asf/cassandra.git
commit 6152cb08dba770f46496d84cde067c6776c66fa0
Merge: 0db0b8f781 7df905a429
Author: Andrés de la Peña <a....@gmail.com>
AuthorDate: Thu Nov 3 11:41:39 2022 +0000
Merge branch 'cassandra-3.11' into cassandra-4.0
.circleci/config-2_1.yml | 539 ++-
.circleci/config-2_1.yml.high_res.patch | 13 +-
.circleci/config-2_1.yml.mid_res.patch | 49 +-
.circleci/config.yml | 5913 +++++++++++++++++++++----------
.circleci/config.yml.HIGHRES | 5499 ++++++++++++++++++++--------
.circleci/config.yml.LOWRES | 5501 ++++++++++++++++++++--------
.circleci/config.yml.MIDRES | 5493 ++++++++++++++++++++--------
.circleci/generate.sh | 17 +-
8 files changed, 16443 insertions(+), 6581 deletions(-)
diff --cc .circleci/config-2_1.yml
index 591cfd71f6,92dd98d332..1e35368e22
--- a/.circleci/config-2_1.yml
+++ b/.circleci/config-2_1.yml
@@@ -163,24 -146,6 +163,30 @@@ j8_seq_executor: &j8_seq_executo
#exec_resource_class: xlarge
parallelism: 1 # sequential, single container tests: no parallelism benefits
+j11_par_executor: &j11_par_executor
+ executor:
+ name: java11-executor
+ #exec_resource_class: xlarge
+ parallelism: 4
+
+j11_small_par_executor: &j11_small_par_executor
+ executor:
+ name: java11-executor
+ #exec_resource_class: xlarge
+ parallelism: 1
+
+j11_small_executor: &j11_small_executor
+ executor:
+ name: java11-executor
+ #exec_resource_class: medium
+ parallelism: 1
+
++j11_seq_executor: &j11_seq_executor
++ executor:
++ name: java11-executor
++ #exec_resource_class: xlarge
++ parallelism: 1 # sequential, single container tests: no parallelism benefits
++
j8_repeated_utest_executor: &j8_repeated_utest_executor
executor:
name: java8-executor
@@@ -242,87 -197,56 +248,171 @@@ j8_separate_jobs: &j8_separate_job
- j8_jvm_dtests_repeat:
requires:
- start_j8_jvm_dtests_repeat
- - build
+ - j8_build
+ - start_j8_cqlshlib_tests:
+ type: approval
+ - j8_cqlshlib_tests:
+ requires:
+ - start_j8_cqlshlib_tests
+ - j8_build
+ # Java 11 unit tests
+ - start_j11_unit_tests:
+ type: approval
+ - j11_unit_tests:
+ requires:
+ - start_j11_unit_tests
+ - j8_build
+ - start_j11_unit_tests_repeat:
+ type: approval
+ - j11_unit_tests_repeat:
+ requires:
+ - start_j11_unit_tests_repeat
+ - j8_build
- # specialized unit tests (all run on request using Java 8)
- - start_utests_long:
+ # specialized unit tests (all run on request)
- - start_utests_long:
++ - start_j8_utests_long:
type: approval
-- - utests_long:
++ - j8_utests_long:
requires:
-- - start_utests_long
- - build
- - start_utests_long_repeat:
++ - start_j8_utests_long
+ - j8_build
- - start_utests_long_repeat:
++ - start_j11_utests_long:
type: approval
-- - utests_long_repeat:
++ - j11_utests_long:
requires:
-- - start_utests_long_repeat
- - build
- - start_utests_cdc:
++ - start_j11_utests_long
+ - j8_build
- - start_utests_compression:
++ - start_j8_utests_long_repeat:
type: approval
- - utests_compression:
- - utests_cdc:
++ - j8_utests_long_repeat:
requires:
- - start_utests_compression
- - start_utests_cdc
- - build
- - start_utests_cdc_repeat:
++ - start_j8_utests_long_repeat
+ - j8_build
- - start_utests_compression_repeat:
++ - start_j11_utests_long_repeat:
type: approval
- - utests_compression_repeat:
- - utests_cdc_repeat:
++ - j11_utests_long_repeat:
requires:
- - start_utests_compression_repeat
- - start_utests_cdc_repeat
- - build
- - start_utests_compression:
++ - start_j11_utests_long_repeat
+ - j8_build
- - start_utests_stress:
++ - start_j8_utests_cdc:
type: approval
- - utests_stress:
- - utests_compression:
++ - j8_utests_cdc:
requires:
- - start_utests_stress
- - start_utests_compression
- - build
- - start_utests_compression_repeat:
++ - start_j8_utests_cdc
+ - j8_build
- - start_utests_stress_repeat:
++ - start_j11_utests_cdc:
type: approval
- - utests_stress_repeat:
- - utests_compression_repeat:
++ - j11_utests_cdc:
requires:
- - start_utests_stress_repeat
- - start_utests_compression_repeat
- - build
- - start_utests_stress:
++ - start_j11_utests_cdc
+ - j8_build
- - start_utests_fqltool:
++ - start_j8_utests_cdc_repeat:
type: approval
- - utests_fqltool:
- - utests_stress:
++ - j8_utests_cdc_repeat:
requires:
- - start_utests_fqltool
- - start_utests_stress
- - build
- - start_utests_stress_repeat:
++ - start_j8_utests_cdc_repeat
+ - j8_build
- - start_utests_fqltool_repeat:
++ - start_j11_utests_cdc_repeat:
+ type: approval
- - utests_fqltool_repeat:
++ - j11_utests_cdc_repeat:
+ requires:
- - start_utests_fqltool_repeat
++ - start_j11_utests_cdc_repeat
+ - j8_build
- - start_utests_system_keyspace_directory:
++ - start_j8_utests_compression:
+ type: approval
- - utests_system_keyspace_directory:
++ - j8_utests_compression:
+ requires:
- - start_utests_system_keyspace_directory
++ - start_j8_utests_compression
++ - j8_build
++ - start_j11_utests_compression:
++ type: approval
++ - j11_utests_compression:
++ requires:
++ - start_j11_utests_compression
++ - j8_build
++ - start_j8_utests_compression_repeat:
++ type: approval
++ - j8_utests_compression_repeat:
++ requires:
++ - start_j8_utests_compression_repeat
++ - j8_build
++ - start_j11_utests_compression_repeat:
++ type: approval
++ - j11_utests_compression_repeat:
++ requires:
++ - start_j11_utests_compression_repeat
++ - j8_build
++ - start_j8_utests_stress:
++ type: approval
++ - j8_utests_stress:
++ requires:
++ - start_j8_utests_stress
++ - j8_build
++ - start_j11_utests_stress:
++ type: approval
++ - j11_utests_stress:
++ requires:
++ - start_j11_utests_stress
++ - j8_build
++ - start_j8_utests_stress_repeat:
+ type: approval
- - utests_stress_repeat:
++ - j8_utests_stress_repeat:
+ requires:
- - start_utests_stress_repeat
- - build
++ - start_j8_utests_stress_repeat
++ - j8_build
++ - start_j11_utests_stress_repeat:
++ type: approval
++ - j11_utests_stress_repeat:
++ requires:
++ - start_j11_utests_stress_repeat
++ - j8_build
++ - start_j8_utests_fqltool:
++ type: approval
++ - j8_utests_fqltool:
++ requires:
++ - start_j8_utests_fqltool
++ - j8_build
++ - start_j11_utests_fqltool:
++ type: approval
++ - j11_utests_fqltool:
++ requires:
++ - start_j11_utests_fqltool
+ - j8_build
- - start_utests_system_keyspace_directory_repeat:
++ - start_j8_utests_fqltool_repeat:
+ type: approval
- - utests_system_keyspace_directory_repeat:
++ - j8_utests_fqltool_repeat:
+ requires:
- - start_utests_system_keyspace_directory_repeat
++ - start_j8_utests_fqltool_repeat
++ - j8_build
++ - start_j11_utests_fqltool_repeat:
++ type: approval
++ - j11_utests_fqltool_repeat:
++ requires:
++ - start_j11_utests_fqltool_repeat
++ - j8_build
++ - start_j8_utests_system_keyspace_directory:
++ type: approval
++ - j8_utests_system_keyspace_directory:
++ requires:
++ - start_j8_utests_system_keyspace_directory
++ - j8_build
++ - start_j11_utests_system_keyspace_directory:
++ type: approval
++ - j11_utests_system_keyspace_directory:
++ requires:
++ - start_j11_utests_system_keyspace_directory
++ - j8_build
++ - start_j8_utests_system_keyspace_directory_repeat:
++ type: approval
++ - j8_utests_system_keyspace_directory_repeat:
++ requires:
++ - start_j8_utests_system_keyspace_directory_repeat
++ - j8_build
++ - start_j11_utests_system_keyspace_directory_repeat:
++ type: approval
++ - j11_utests_system_keyspace_directory_repeat:
++ requires:
++ - start_j11_utests_system_keyspace_directory_repeat
+ - j8_build
- start_j8_dtest_jars_build:
type: approval
- j8_dtest_jars_build:
@@@ -483,73 -321,57 +573,133 @@@ j8_pre-commit_jobs: &j8_pre-commit_job
# Java 8 unit tests
- j8_unit_tests:
requires:
- - build
+ - j8_build
- j8_unit_tests_repeat:
requires:
- - build
+ - j8_build
- j8_jvm_dtests:
requires:
- - build
+ - j8_build
- j8_jvm_dtests_repeat:
requires:
- - build
+ - j8_build
+ - j8_cqlshlib_tests:
+ requires:
+ - j8_build
+ # Java 11 unit tests
+ - j11_unit_tests:
+ requires:
+ - j8_build
+ - j11_unit_tests_repeat:
+ requires:
+ - j8_build
- # specialized unit tests (all run on request using Java 8)
+ # specialized unit tests (all run on request)
- start_utests_long:
type: approval
-- - utests_long:
++ - j8_utests_long:
+ requires:
+ - start_utests_long
+ - j8_build
- - utests_long_repeat:
++ - j11_utests_long:
+ requires:
+ - start_utests_long
+ - j8_build
++ - j8_utests_long_repeat:
+ requires:
+ - start_utests_long
- - build
- - utests_long_repeat:
++ - j8_build
++ - j11_utests_long_repeat:
+ requires:
+ - start_utests_long
- - build
++ - j8_build
+ - start_utests_cdc:
+ type: approval
- - utests_cdc:
++ - j8_utests_cdc:
++ requires:
++ - start_utests_cdc
++ - j8_build
++ - j11_utests_cdc:
+ requires:
+ - start_utests_cdc
- - build
- - utests_cdc_repeat:
++ - j8_build
++ - j8_utests_cdc_repeat:
+ requires:
+ - start_utests_cdc
- - build
++ - j8_build
++ - j11_utests_cdc_repeat:
++ requires:
++ - start_utests_cdc
++ - j8_build
- start_utests_compression:
type: approval
-- - utests_compression:
++ - j8_utests_compression:
+ requires:
+ - start_utests_compression
+ - j8_build
- - utests_compression_repeat:
++ - j11_utests_compression:
+ requires:
+ - start_utests_compression
- - build
- - utests_compression_repeat:
++ - j8_build
++ - j8_utests_compression_repeat:
+ requires:
+ - start_utests_compression
- - build
++ - j8_build
++ - j11_utests_compression_repeat:
+ requires:
+ - start_utests_compression
+ - j8_build
- start_utests_stress:
type: approval
-- - utests_stress:
++ - j8_utests_stress:
++ requires:
++ - start_utests_stress
++ - j8_build
++ - j11_utests_stress:
+ requires:
+ - start_utests_stress
- - build
- - utests_stress_repeat:
++ - j8_build
++ - j8_utests_stress_repeat:
requires:
- start_utests_stress
- - build
+ - j8_build
- - utests_stress_repeat:
++ - j11_utests_stress_repeat:
+ requires:
+ - start_utests_stress
+ - j8_build
+ - start_utests_fqltool:
+ type: approval
- - utests_fqltool:
++ - j8_utests_fqltool:
+ requires:
+ - start_utests_fqltool
+ - j8_build
- - utests_fqltool_repeat:
++ - j11_utests_fqltool:
+ requires:
+ - start_utests_fqltool
+ - j8_build
- - utests_system_keyspace_directory:
++ - j8_utests_fqltool_repeat:
+ requires:
++ - start_utests_fqltool
+ - j8_build
- - utests_system_keyspace_directory_repeat:
++ - j11_utests_fqltool_repeat:
+ requires:
++ - start_utests_fqltool
++ - j8_build
++ - start_utests_system_keyspace_directory:
++ type: approval
++ - j8_utests_system_keyspace_directory:
++ requires:
++ - j8_build
++ - j11_utests_system_keyspace_directory:
++ requires:
++ - start_utests_system_keyspace_directory
++ - j8_build
++ - j8_utests_system_keyspace_directory_repeat:
++ requires:
++ - j8_build
++ - j11_utests_system_keyspace_directory_repeat:
++ requires:
++ - start_utests_system_keyspace_directory
+ - j8_build
- start_jvm_upgrade_dtests:
type: approval
- j8_dtest_jars_build:
@@@ -593,206 -402,16 +743,340 @@@
type: approval
- j8_upgrade_dtests:
requires:
+ - j8_build
- start_upgrade_tests
- - build
- j8_upgrade_dtests_repeat:
requires:
+ - j8_build
- start_upgrade_tests
- - build
- # Java 8 repeated utest
- - j8_repeated_ant_test:
+ # Java 8 cqlsh dtests
+ - j8_cqlsh-dtests-py2-with-vnodes:
+ requires:
+ - j8_build
+ - j8_cqlsh_dtests_py3_vnode:
+ requires:
+ - j8_build
+ - j8_cqlsh_dtests_py38_vnode:
+ requires:
+ - j8_build
+ - j8_cqlsh-dtests-py2-no-vnodes:
+ requires:
+ - j8_build
+ - j8_cqlsh_dtests_py3:
+ requires:
+ - j8_build
+ - j8_cqlsh_dtests_py38:
+ requires:
+ - j8_build
+ # Java 11 cqlsh dtests
+ - j11_cqlsh-dtests-py2-with-vnodes:
+ requires:
+ - j8_build
+ - j11_cqlsh_dtests_py3_vnode:
requires:
- - build
+ - j8_build
+ - j11_cqlsh_dtests_py38_vnode:
+ requires:
+ - j8_build
+ - j11_cqlsh-dtests-py2-no-vnodes:
+ requires:
+ - j8_build
+ - j11_cqlsh_dtests_py3:
+ requires:
+ - j8_build
+ - j11_cqlsh_dtests_py38:
+ requires:
+ - j8_build
+
+j11_separate_jobs: &j11_separate_jobs
+ jobs:
+ - start_j11_build:
+ type: approval
+ - j11_build:
+ requires:
+ - start_j11_build
+ # Java 11 unit tests
+ - start_j11_unit_tests:
+ type: approval
+ - j11_unit_tests:
+ requires:
+ - start_j11_unit_tests
+ - j11_build
+ - start_j11_unit_tests_repeat:
+ type: approval
+ - j11_unit_tests_repeat:
+ requires:
+ - start_j11_unit_tests_repeat
+ - j11_build
+ - start_j11_jvm_dtests:
+ type: approval
+ - j11_jvm_dtests:
+ requires:
+ - start_j11_jvm_dtests
+ - j11_build
+ - start_j11_jvm_dtests_repeat:
+ type: approval
+ - j11_jvm_dtests_repeat:
+ requires:
+ - start_j11_jvm_dtests_repeat
+ - j11_build
+ - start_j11_cqlshlib_tests:
+ type: approval
+ - j11_cqlshlib_tests:
+ requires:
+ - start_j11_cqlshlib_tests
+ - j11_build
+ # Java 11 dtests
+ - start_j11_dtests:
+ type: approval
+ - j11_dtests:
+ requires:
+ - start_j11_dtests
+ - j11_build
+ - start_j11_dtests_vnode:
+ type: approval
+ - j11_dtests_vnode:
+ requires:
+ - start_j11_dtests_vnode
+ - j11_build
+ - start_j11_dtests_repeat:
+ type: approval
+ - j11_dtests_repeat:
+ requires:
+ - start_j11_dtests_repeat
+ - j11_build
+ - start_j11_dtests_vnode_repeat:
+ type: approval
+ - j11_dtests_vnode_repeat:
+ requires:
+ - start_j11_dtests_vnode_repeat
+ - j11_build
+ - start_j11_cqlsh_tests:
+ type: approval
+ - j11_cqlsh-dtests-py2-with-vnodes:
+ requires:
+ - start_j11_cqlsh_tests
+ - j11_build
+ - j11_cqlsh_dtests_py3_vnode:
+ requires:
+ - start_j11_cqlsh_tests
+ - j11_build
+ - j11_cqlsh_dtests_py38_vnode:
+ requires:
+ - start_j11_cqlsh_tests
+ - j11_build
+ - j11_cqlsh-dtests-py2-no-vnodes:
+ requires:
+ - start_j11_cqlsh_tests
+ - j11_build
+ - j11_cqlsh_dtests_py3:
+ requires:
+ - start_j11_cqlsh_tests
+ - j11_build
+ - j11_cqlsh_dtests_py38:
+ requires:
+ - start_j11_cqlsh_tests
+ - j11_build
++ # specialized unit tests (all run on request)
++ - start_j11_utests_long:
++ type: approval
++ - j11_utests_long:
++ requires:
++ - start_j11_utests_long
++ - j11_build
++ - start_j11_utests_long_repeat:
++ type: approval
++ - j11_utests_long_repeat:
++ requires:
++ - start_j11_utests_long_repeat
++ - j11_build
++ - start_j11_utests_cdc:
++ type: approval
++ - j11_utests_cdc:
++ requires:
++ - start_j11_utests_cdc
++ - j11_build
++ - start_j11_utests_cdc_repeat:
++ type: approval
++ - j11_utests_cdc_repeat:
++ requires:
++ - start_j11_utests_cdc_repeat
++ - j11_build
++ - start_j11_utests_compression:
++ type: approval
++ - j11_utests_compression:
++ requires:
++ - start_j11_utests_compression
++ - j11_build
++ - start_j11_utests_compression_repeat:
++ type: approval
++ - j11_utests_compression_repeat:
++ requires:
++ - start_j11_utests_compression_repeat
++ - j11_build
++ - start_j11_utests_stress:
++ type: approval
++ - j11_utests_stress:
++ requires:
++ - start_j11_utests_stress
++ - j11_build
++ - start_j11_utests_stress_repeat:
++ type: approval
++ - j11_utests_stress_repeat:
++ requires:
++ - start_j11_utests_stress_repeat
++ - j11_build
++ - start_j11_utests_fqltool:
++ type: approval
++ - j11_utests_fqltool:
++ requires:
++ - start_j11_utests_fqltool
++ - j11_build
++ - start_j11_utests_fqltool_repeat:
++ type: approval
++ - j11_utests_fqltool_repeat:
++ requires:
++ - start_j11_utests_fqltool_repeat
++ - j11_build
++ - start_j11_utests_system_keyspace_directory:
++ type: approval
++ - j11_utests_system_keyspace_directory:
++ requires:
++ - start_j11_utests_system_keyspace_directory
++ - j11_build
++ - start_j11_utests_system_keyspace_directory_repeat:
++ type: approval
++ - j11_utests_system_keyspace_directory_repeat:
++ requires:
++ - start_j11_utests_system_keyspace_directory_repeat
++ - j11_build
+ # Java 11 repeated utest
+ - start_j11_repeated_ant_test:
+ type: approval
+ - j11_repeated_ant_test:
+ requires:
+ - start_j11_repeated_ant_test
+ - j11_build
+
+j11_pre-commit_jobs: &j11_pre-commit_jobs
+ jobs:
+ - start_pre-commit_tests:
+ type: approval
+ - j11_build:
+ requires:
+ - start_pre-commit_tests
+ - j11_unit_tests:
+ requires:
+ - j11_build
+ - j11_unit_tests_repeat:
+ requires:
+ - j11_build
+ - j11_jvm_dtests:
+ requires:
+ - j11_build
+ - j11_jvm_dtests_repeat:
+ requires:
+ - j11_build
+ - j11_cqlshlib_tests:
+ requires:
+ - j11_build
+ - j11_jvm_dtests:
+ requires:
+ - j11_build
+ - j11_cqlshlib_tests:
+ requires:
+ - j11_build
+ - j11_dtests:
+ requires:
+ - j11_build
+ - j11_dtests_repeat:
+ requires:
+ - j11_build
+ - j11_dtests_vnode:
+ requires:
+ - j11_build
+ - j11_dtests_vnode_repeat:
+ requires:
+ - j11_build
+ - j11_cqlsh-dtests-py2-with-vnodes:
+ requires:
+ - j11_build
+ - j11_cqlsh_dtests_py3_vnode:
+ requires:
+ - j11_build
+ - j11_cqlsh_dtests_py38_vnode:
+ requires:
+ - j11_build
+ - j11_cqlsh-dtests-py2-no-vnodes:
+ requires:
+ - j11_build
+ - j11_cqlsh_dtests_py3:
+ requires:
+ - j11_build
+ - j11_cqlsh_dtests_py38:
+ requires:
+ - j11_build
++ # specialized unit tests (all run on request)
++ - start_utests_long:
++ type: approval
++ - j11_utests_long:
++ requires:
++ - start_utests_long
++ - j11_build
++ - j11_utests_long_repeat:
++ requires:
++ - start_utests_long
++ - j11_build
++ - start_utests_cdc:
++ type: approval
++ - j11_utests_cdc:
++ requires:
++ - start_utests_cdc
++ - j11_build
++ - j11_utests_cdc_repeat:
++ requires:
++ - start_utests_cdc
++ - j11_build
++ - start_utests_compression:
++ type: approval
++ - j11_utests_compression:
++ requires:
++ - start_utests_compression
++ - j11_build
++ - j11_utests_compression_repeat:
++ requires:
++ - start_utests_compression
++ - j11_build
++ - start_utests_stress:
++ type: approval
++ - j11_utests_stress:
++ requires:
++ - start_utests_stress
++ - j11_build
++ - j11_utests_stress_repeat:
++ requires:
++ - start_utests_stress
++ - j11_build
++ - start_utests_fqltool:
++ type: approval
++ - j11_utests_fqltool:
++ requires:
++ - start_utests_fqltool
++ - j11_build
++ - j11_utests_fqltool_repeat:
++ requires:
++ - start_utests_fqltool
++ - j11_build
++ - start_utests_system_keyspace_directory:
++ type: approval
++ - j11_utests_system_keyspace_directory:
++ requires:
++ - start_utests_system_keyspace_directory
++ - j11_build
++ - j11_utests_system_keyspace_directory_repeat:
++ requires:
++ - start_utests_system_keyspace_directory
++ - j11_build
workflows:
version: 2
@@@ -817,231 -434,24 +1101,295 @@@ executors
JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ java11-executor:
+ parameters:
+ exec_resource_class:
+ type: string
+ default: medium
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: << parameters.exec_resource_class >>
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ environment:
+ <<: *default_env_vars
+ JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ CASSANDRA_USE_JDK11: true
+
+build_common: &build_common
+ parallelism: 1 # This job doesn't benefit from parallelism
+ steps:
+ - log_environment
+ - clone_cassandra
+ - build_cassandra
+ - run_eclipse_warnings
+ - persist_to_workspace:
+ root: /home/cassandra
+ paths:
+ - cassandra
+ - .m2
+
jobs:
- build:
+ j8_build:
+ executor: java8-executor
+ <<: *build_common
+
+ j11_build:
+ executor: java11-executor
+ <<: *build_common
+
+ j8_dtest_jars_build:
executor: java8-executor
- parallelism: 1 # This job doesn't benefit from parallelism
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - build_cassandra_dtest_jars
+ - persist_to_workspace:
+ root: /home/cassandra
+ paths:
+ - dtest_jars
+
+ j8_unit_tests:
+ <<: *j8_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - create_junit_containers
+ - log_environment
+ - run_parallel_junit_tests
+
+ j8_jvm_dtests:
+ <<: *j8_small_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - create_junit_containers:
+ classlistprefix: distributed
+ extra_filters: "| grep -v upgrade"
+ - log_environment
+ - run_parallel_junit_tests:
+ classlistprefix: distributed
+ target: "testclasslist"
+
+ j11_jvm_dtests:
+ <<: *j11_small_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - create_junit_containers:
+ classlistprefix: distributed
+ extra_filters: "| grep -v upgrade"
+ - log_environment
+ - run_parallel_junit_tests:
+ classlistprefix: distributed
+ target: "testclasslist"
+
+ j8_jvm_upgrade_dtests:
+ <<: *j8_medium_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - create_junit_containers:
+ classlistprefix: distributed
+ extra_filters: "| grep upgrade"
+ - log_environment
+ - run_parallel_junit_tests:
+ classlistprefix: distributed
+ target: "testclasslist"
+
+ j11_unit_tests:
+ <<: *j11_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - create_junit_containers
+ - log_environment
+ - run_parallel_junit_tests
+
+ j8_cqlshlib_tests:
+ <<: *j8_small_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run_cqlshlib_tests
+
+ j11_cqlshlib_tests:
+ <<: *j11_small_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run_cqlshlib_tests
+
- utests_long:
++ j8_utests_long:
+ <<: *j8_seq_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run_junit_tests:
+ target: long-test
+
- utests_compression:
++ j11_utests_long:
++ <<: *j11_seq_executor
++ steps:
++ - attach_workspace:
++ at: /home/cassandra
++ - run_junit_tests:
++ target: long-test
++
++ j8_utests_cdc:
++ <<: *j8_par_executor
++ steps:
++ - attach_workspace:
++ at: /home/cassandra
++ - create_junit_containers
++ - log_environment
++ - run_parallel_junit_tests:
++ target: testclasslist-cdc
++
++ j11_utests_cdc:
++ <<: *j11_par_executor
++ steps:
++ - attach_workspace:
++ at: /home/cassandra
++ - create_junit_containers
++ - log_environment
++ - run_parallel_junit_tests:
++ target: testclasslist-cdc
++
++ j8_utests_compression:
+ <<: *j8_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - create_junit_containers
+ - log_environment
+ - run_parallel_junit_tests:
+ target: testclasslist-compression
+
- utests_stress:
++ j11_utests_compression:
++ <<: *j11_par_executor
++ steps:
++ - attach_workspace:
++ at: /home/cassandra
++ - create_junit_containers
++ - log_environment
++ - run_parallel_junit_tests:
++ target: testclasslist-compression
++
++ j8_utests_stress:
+ <<: *j8_seq_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run_junit_tests:
+ target: stress-test
+
- utests_fqltool:
++ j11_utests_stress:
++ <<: *j11_seq_executor
++ steps:
++ - attach_workspace:
++ at: /home/cassandra
++ - run_junit_tests:
++ target: stress-test
++
++ j8_utests_fqltool:
++ <<: *j8_seq_executor
++ steps:
++ - attach_workspace:
++ at: /home/cassandra
++ - run_junit_tests:
++ target: fqltool-test
++
++ j11_utests_fqltool:
+ <<: *j8_seq_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run_junit_tests:
+ target: fqltool-test
+
- utests_system_keyspace_directory:
++ j8_utests_system_keyspace_directory:
+ <<: *j8_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - create_junit_containers
+ - log_environment
+ - run_parallel_junit_tests:
+ target: testclasslist-system-keyspace-directory
+
++ j11_utests_system_keyspace_directory:
++ <<: *j11_par_executor
++ steps:
++ - attach_workspace:
++ at: /home/cassandra
++ - create_junit_containers
++ - log_environment
++ - run_parallel_junit_tests:
++ target: testclasslist-system-keyspace-directory
++
+ j8_dtests_vnode:
+ <<: *j8_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - clone_dtest
+ - create_venv
+ - create_dtest_containers:
+ file_tag: j8_with_vnodes
+ run_dtests_extra_args: "--use-vnodes --skip-resource-intensive-tests --pytest-options '-k not cql'"
+ - run_dtests:
+ file_tag: j8_with_vnodes
+ pytest_extra_args: '--use-vnodes --num-tokens=16 --skip-resource-intensive-tests'
+
+ j11_dtests_vnode:
+ <<: *j11_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - log_environment
+ - clone_dtest
+ - create_venv
+ - create_dtest_containers:
+ file_tag: j11_with_vnodes
+ run_dtests_extra_args: "--use-vnodes --skip-resource-intensive-tests --pytest-options '-k not cql'"
+ - run_dtests:
+ file_tag: j11_with_vnodes
+ pytest_extra_args: '--use-vnodes --num-tokens=16 --skip-resource-intensive-tests'
+
+ j8_dtests:
+ <<: *j8_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - clone_dtest
+ - create_venv
+ - create_dtest_containers:
+ file_tag: j8_without_vnodes
+ run_dtests_extra_args: "--skip-resource-intensive-tests --pytest-options '-k not cql'"
+ - run_dtests:
+ file_tag: j8_without_vnodes
+ pytest_extra_args: '--skip-resource-intensive-tests'
+
+ j11_dtests:
+ <<: *j11_par_executor
steps:
- - log_environment
- - clone_cassandra
- - build_cassandra
- - run_eclipse_warnings
- - persist_to_workspace:
- root: /home/cassandra
- paths:
- - cassandra
- - .m2
+ - attach_workspace:
+ at: /home/cassandra
+ - log_environment
+ - clone_dtest
+ - create_venv
+ - create_dtest_containers:
+ file_tag: j11_without_vnodes
+ run_dtests_extra_args: "--skip-resource-intensive-tests --pytest-options '-k not cql'"
+ - run_dtests:
+ file_tag: j11_without_vnodes
+ pytest_extra_args: '--skip-resource-intensive-tests'
- j8_dtest_jars_build:
- executor: java8-executor
- parallelism: 1
+ j8_upgrade_dtests:
+ <<: *j8_par_executor
steps:
- attach_workspace:
at: /home/cassandra
@@@ -1254,7 -583,15 +1602,31 @@@
- log_environment
- run_unit_tests_repeat
- utests_compression_repeat:
- utests_cdc_repeat:
++ j11_unit_tests_repeat:
++ <<: *j11_repeated_utest_executor
++ steps:
++ - attach_workspace:
++ at: /home/cassandra
++ - log_environment
++ - run_unit_tests_repeat
++
++ j8_utests_cdc_repeat:
+ <<: *j8_repeated_utest_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - log_environment
+ - run_utests_cdc_repeat
+
- utests_compression_repeat:
++ j11_utests_cdc_repeat:
++ <<: *j11_repeated_utest_executor
++ steps:
++ - attach_workspace:
++ at: /home/cassandra
++ - log_environment
++ - run_utests_cdc_repeat
++
++ j8_utests_compression_repeat:
<<: *j8_repeated_utest_executor
steps:
- attach_workspace:
@@@ -1262,23 -599,7 +1634,47 @@@
- log_environment
- run_utests_compression_repeat
- utests_system_keyspace_directory_repeat:
- utests_long_repeat:
++ j11_utests_compression_repeat:
++ <<: *j11_repeated_utest_executor
++ steps:
++ - attach_workspace:
++ at: /home/cassandra
++ - log_environment
++ - run_utests_compression_repeat
++
++ j8_utests_system_keyspace_directory_repeat:
+ <<: *j8_repeated_utest_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - log_environment
+ - run_utests_system_keyspace_directory_repeat
+
- utests_fqltool_repeat:
++ j11_utests_system_keyspace_directory_repeat:
++ <<: *j11_repeated_utest_executor
++ steps:
++ - attach_workspace:
++ at: /home/cassandra
++ - log_environment
++ - run_utests_system_keyspace_directory_repeat
++
++ j8_utests_fqltool_repeat:
+ <<: *j8_repeated_utest_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - log_environment
+ - run_utests_fqltool_repeat
+
- utests_long_repeat:
++ j11_utests_fqltool_repeat:
++ <<: *j11_repeated_utest_executor
++ steps:
++ - attach_workspace:
++ at: /home/cassandra
++ - log_environment
++ - run_utests_fqltool_repeat
++
++ j8_utests_long_repeat:
<<: *j8_repeated_utest_executor
steps:
- attach_workspace:
@@@ -1286,7 -607,7 +1682,15 @@@
- log_environment
- run_utests_long_repeat
-- utests_stress_repeat:
++ j11_utests_long_repeat:
++ <<: *j11_repeated_utest_executor
++ steps:
++ - attach_workspace:
++ at: /home/cassandra
++ - log_environment
++ - run_utests_long_repeat
++
++ j8_utests_stress_repeat:
<<: *j8_repeated_utest_executor
steps:
- attach_workspace:
@@@ -1294,14 -615,6 +1698,14 @@@
- log_environment
- run_utests_stress_repeat
- j11_unit_tests_repeat:
++ j11_utests_stress_repeat:
+ <<: *j11_repeated_utest_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - log_environment
- - run_unit_tests_repeat
++ - run_utests_stress_repeat
+
j8_jvm_dtests_repeat:
<<: *j8_repeated_utest_executor
steps:
@@@ -1865,7 -1095,16 +2277,18 @@@ commands
# Put manually specified tests and automatically detected tests together, removing duplicates
tests=$(echo <<parameters.tests>> | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
echo "Tests to be repeated: ${tests}"
-
+
+ # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
+ target=<<parameters.target>>
+ testtag=""
+ if [[ $target == "test-cdc" ]]; then
+ testtag="cdc"
+ elif [[ $target == "test-compression" ]]; then
+ testtag="compression"
++ elif [[ $target == "test-system-keyspace-directory" ]]; then
++ testtag="system_keyspace_directory"
+ fi
+
# Run each test class as many times as requested.
exit_code="$?"
for test in $tests; do
@@@ -1892,9 -1129,9 +2315,9 @@@
else
name_arg="-Dtest.name=$class"
fi
-
+
# Prepare the -Dtest.methods argument, which is optional
- if [ $method == "" ]; then
+ if [[ $method == "" ]]; then
methods_arg=""
else
methods_arg="-Dtest.methods=$method"
diff --cc .circleci/config-2_1.yml.high_res.patch
index 039203ddd2,6c1ca83ea4..c8b629b095
--- a/.circleci/config-2_1.yml.high_res.patch
+++ b/.circleci/config-2_1.yml.high_res.patch
@@@ -1,6 -1,17 +1,6 @@@
- --- config-2_1.yml 2022-10-15 13:31:58.877152008 +0100
- +++ config-2_1.yml.HIGHRES 2022-10-15 13:32:14.947560988 +0100
---- config-2_1.yml 2022-10-27 10:16:44.059665934 +0100
-+++ config-2_1.yml.HIGHRES 2022-10-27 10:18:01.956045811 +0100
-@@ -40,8 +40,8 @@
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
-- CCM_MAX_HEAP_SIZE: 1024M
-- CCM_HEAP_NEWSIZE: 256M
-+ CCM_MAX_HEAP_SIZE: 2048M
-+ CCM_HEAP_NEWSIZE: 512M
-
- # Whether the repeated test iterations should stop on the first failure by default.
- REPEATED_TESTS_STOP_ON_FAILURE: false
-@@ -125,46 +125,50 @@
++--- config-2_1.yml 2022-10-27 10:22:41.868009084 +0100
+++++ config-2_1.yml.HIGHRES 2022-10-27 10:23:05.374865893 +0100
+@@ -136,14 +136,14 @@
j8_par_executor: &j8_par_executor
executor:
name: java8-executor
@@@ -15,11 -26,8 +15,11 @@@
- #exec_resource_class: xlarge
- parallelism: 1
+ exec_resource_class: xlarge
-+ parallelism: 2
++ parallelism: 5
+ j8_small_executor: &j8_small_executor
+ executor:
- @@ -154,62 +154,68 @@
++@@ -154,68 +154,74 @@
j8_medium_par_executor: &j8_medium_par_executor
executor:
name: java8-executor
@@@ -32,34 -40,9 +32,41 @@@
executor:
name: java8-executor
- #exec_resource_class: xlarge
++ exec_resource_class: xlarge
+ parallelism: 1 # sequential, single container tests: no parallelism benefits
+
+ j11_par_executor: &j11_par_executor
+ executor:
+ name: java11-executor
+- #exec_resource_class: xlarge
+- parallelism: 4
++ exec_resource_class: xlarge
++ parallelism: 100
+
+-j11_small_par_executor: &j11_small_par_executor
++j11_small_executor: &j11_small_executor
+ executor:
+ name: java11-executor
+- #exec_resource_class: xlarge
++ exec_resource_class: medium
+ parallelism: 1
+
+-j11_small_executor: &j11_small_executor
++j11_small_par_executor: &j11_small_par_executor
+ executor:
+ name: java11-executor
+- #exec_resource_class: medium
+- parallelism: 1
++ exec_resource_class: xlarge
++ parallelism: 5
+
++ j11_seq_executor: &j11_seq_executor
++ executor:
++ name: java11-executor
++- #exec_resource_class: xlarge
+ + exec_resource_class: xlarge
+ parallelism: 1 # sequential, single container tests: no parallelism benefits
+
j8_repeated_utest_executor: &j8_repeated_utest_executor
executor:
name: java8-executor
diff --cc .circleci/config-2_1.yml.mid_res.patch
index 52e74e2ecb,8fce189ce8..9fe0c53a7f
--- a/.circleci/config-2_1.yml.mid_res.patch
+++ b/.circleci/config-2_1.yml.mid_res.patch
@@@ -1,10 -1,9 +1,10 @@@
- --- config-2_1.yml 2022-10-15 13:31:58.877152008 +0100
- +++ config-2_1.yml.MIDRES 2022-10-15 13:32:14.940950632 +0100
---- config-2_1.yml 2022-10-27 10:16:44.059665934 +0100
-+++ config-2_1.yml.MIDRES 2022-10-27 10:18:01.949746099 +0100
-@@ -126,45 +126,65 @@
++--- config-2_1.yml 2022-10-27 10:22:41.868009084 +0100
+++++ config-2_1.yml.MIDRES 2022-10-27 10:23:05.368183086 +0100
+@@ -136,14 +136,14 @@
+ j8_par_executor: &j8_par_executor
executor:
name: java8-executor
- #exec_resource_class: xlarge
+- #exec_resource_class: xlarge
- parallelism: 4
+ exec_resource_class: medium
+ parallelism: 25
@@@ -43,38 -39,10 +43,45 @@@
j8_seq_executor: &j8_seq_executor
executor:
name: java8-executor
- #exec_resource_class: xlarge
+- #exec_resource_class: xlarge
++ exec_resource_class: medium
+ parallelism: 1 # sequential, single container tests: no parallelism benefits
+
+ j11_par_executor: &j11_par_executor
+ executor:
+ name: java11-executor
+- #exec_resource_class: xlarge
+- parallelism: 4
++ exec_resource_class: medium
++ parallelism: 25
+
+ j11_small_par_executor: &j11_small_par_executor
+ executor:
+ name: java11-executor
+- #exec_resource_class: xlarge
+- parallelism: 1
++ #exec_resource_class: large
++ parallelism: 10
+
+ j11_small_executor: &j11_small_executor
+ executor:
- @@ -181,35 +193,47 @@
++@@ -181,41 +193,53 @@
+ #exec_resource_class: medium
+ parallelism: 1
+
++j11_large_par_executor: &j11_large_par_executor
++ executor:
++ name: java11-executor
++ exec_resource_class: large
++ parallelism: 50
++
++ j11_seq_executor: &j11_seq_executor
++ executor:
++ name: java11-executor
++- #exec_resource_class: xlarge
+ + exec_resource_class: medium
+ parallelism: 1 # sequential, single container tests: no parallelism benefits
+
j8_repeated_utest_executor: &j8_repeated_utest_executor
executor:
name: java8-executor
@@@ -103,24 -71,10 +110,24 @@@
+ exec_resource_class: large
+ parallelism: 25
- separate_jobs: &separate_jobs
+ j11_repeated_utest_executor: &j11_repeated_utest_executor
+ executor:
+ name: java11-executor
+- parallelism: 4
++ exec_resource_class: medium
++ parallelism: 25
+
+ j11_repeated_dtest_executor: &j11_repeated_dtest_executor
+ executor:
+ name: java11-executor
+- parallelism: 4
++ exec_resource_class: large
++ parallelism: 25
+
+ j8_separate_jobs: &j8_separate_jobs
jobs:
- @@ -983,7 +1007,7 @@
-@@ -531,7 +551,7 @@
- target: stress-test
++@@ -1331,7 +1355,7 @@
+ target: testclasslist-system-keyspace-directory
j8_dtests_vnode:
- <<: *j8_par_executor
@@@ -128,17 -82,8 +135,17 @@@
steps:
- attach_workspace:
at: /home/cassandra
- @@ -997,7 +1021,7 @@
-@@ -545,7 +565,7 @@
- pytest_extra_args: '--use-vnodes --num-tokens=32 --skip-resource-intensive-tests'
++@@ -1345,7 +1369,7 @@
+ pytest_extra_args: '--use-vnodes --num-tokens=16 --skip-resource-intensive-tests'
+
+ j11_dtests_vnode:
+- <<: *j11_par_executor
++ <<: *j11_large_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
- @@ -1012,7 +1036,7 @@
++@@ -1360,7 +1384,7 @@
+ pytest_extra_args: '--use-vnodes --num-tokens=16 --skip-resource-intensive-tests'
j8_dtests:
- <<: *j8_par_executor
@@@ -146,134 -91,12 +153,134 @@@
steps:
- attach_workspace:
at: /home/cassandra
- @@ -1026,7 +1050,7 @@
-@@ -559,7 +579,7 @@
++@@ -1374,7 +1398,7 @@
pytest_extra_args: '--skip-resource-intensive-tests'
+ j11_dtests:
+- <<: *j11_par_executor
++ <<: *j11_large_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
- @@ -1041,7 +1065,7 @@
++@@ -1389,7 +1413,7 @@
+ pytest_extra_args: '--skip-resource-intensive-tests'
+
j8_upgrade_dtests:
- <<: *j8_par_executor
+ <<: *j8_very_large_par_executor
steps:
- attach_workspace:
at: /home/cassandra
- @@ -1055,7 +1079,7 @@
++@@ -1403,7 +1427,7 @@
+ pytest_extra_args: '--execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all'
+
+ j8_cqlsh-dtests-py2-with-vnodes:
+- <<: *j8_par_executor
++ <<: *j8_large_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
- @@ -1070,7 +1094,7 @@
++@@ -1418,7 +1442,7 @@
+ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python2.7'
+
+ j8_cqlsh_dtests_py3_vnode:
+- <<: *j8_par_executor
++ <<: *j8_large_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
- @@ -1085,7 +1109,7 @@
++@@ -1433,7 +1457,7 @@
+ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
+
+ j8_cqlsh_dtests_py38_vnode:
+- <<: *j8_par_executor
++ <<: *j8_large_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
- @@ -1103,7 +1127,7 @@
++@@ -1451,7 +1475,7 @@
+ python_version: '3.8'
+
+ j8_cqlsh-dtests-py2-no-vnodes:
+- <<: *j8_par_executor
++ <<: *j8_large_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
- @@ -1118,7 +1142,7 @@
++@@ -1466,7 +1490,7 @@
+ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python2.7'
+
+ j8_cqlsh_dtests_py3:
+- <<: *j8_par_executor
++ <<: *j8_large_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
- @@ -1133,7 +1157,7 @@
++@@ -1481,7 +1505,7 @@
+ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
+
+ j8_cqlsh_dtests_py38:
+- <<: *j8_par_executor
++ <<: *j8_large_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
- @@ -1151,7 +1175,7 @@
++@@ -1499,7 +1523,7 @@
+ python_version: '3.8'
+
+ j11_cqlsh-dtests-py2-with-vnodes:
+- <<: *j11_par_executor
++ <<: *j11_large_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
- @@ -1166,7 +1190,7 @@
++@@ -1514,7 +1538,7 @@
+ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python2.7'
+
+ j11_cqlsh_dtests_py3_vnode:
+- <<: *j11_par_executor
++ <<: *j11_large_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
- @@ -1181,7 +1205,7 @@
++@@ -1529,7 +1553,7 @@
+ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
+
+ j11_cqlsh_dtests_py38_vnode:
+- <<: *j11_par_executor
++ <<: *j11_large_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
- @@ -1199,7 +1223,7 @@
++@@ -1547,7 +1571,7 @@
+ python_version: '3.8'
+
+ j11_cqlsh-dtests-py2-no-vnodes:
+- <<: *j11_par_executor
++ <<: *j11_large_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
- @@ -1214,7 +1238,7 @@
++@@ -1562,7 +1586,7 @@
+ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python2.7'
+
+ j11_cqlsh_dtests_py3:
+- <<: *j11_par_executor
++ <<: *j11_large_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
- @@ -1229,7 +1253,7 @@
++@@ -1577,7 +1601,7 @@
+ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
+
+ j11_cqlsh_dtests_py38:
+- <<: *j11_par_executor
++ <<: *j11_large_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
- @@ -2156,3 +2180,4 @@
++@@ -2579,3 +2603,4 @@
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_logs
++
diff --cc .circleci/config.yml
index afeb6e8288,aa877ba101..3bd389e0a3
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@@ -132,9 -129,9 +132,9 @@@ jobs
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
-- utests_compression_repeat:
++ j11_utests_fqltool_repeat:
docker:
-- - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
++ - image: apache/cassandra-testing-ubuntu2004-java11:latest
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
@@@ -177,15 -174,24 +177,26 @@@
fi
# Calculate the number of test iterations to be run by the current parallel runner.
-- count=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))
-- if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
++ count=$((${REPEATED_UTESTS_FQLTOOL_COUNT} / CIRCLE_NODE_TOTAL))
++ if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_FQLTOOL_COUNT} % CIRCLE_NODE_TOTAL))); then
count=$((count+1))
fi
# Put manually specified tests and automatically detected tests together, removing duplicates
-- tests=$(echo ${REPEATED_UTESTS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
++ tests=$(echo ${REPEATED_UTESTS_FQLTOOL} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
echo "Tests to be repeated: ${tests}"
+ # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
- target=test-compression
++ target=fqltool-test
+ testtag=""
+ if [[ $target == "test-cdc" ]]; then
+ testtag="cdc"
+ elif [[ $target == "test-compression" ]]; then
+ testtag="compression"
++ elif [[ $target == "test-system-keyspace-directory" ]]; then
++ testtag="system_keyspace_directory"
+ fi
+
# Run each test class as many times as requested.
exit_code="$?"
for test in $tests; do
@@@ -226,7 -230,7 +237,7 @@@
# run the test
status="passes"
if !( set -o pipefail && \
-- ant test-compression $name_arg $methods_arg -Dno-build-test=true | \
++ ant fqltool-test $name_arg $methods_arg -Dno-build-test=true | \
tee stdout.txt \
); then
status="fails"
@@@ -309,9 -310,9 +320,10 @@@
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
- REPEATED_ANT_TEST_COUNT: 500
-- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
-- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- utests_stress_repeat:
++ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - CASSANDRA_USE_JDK11: true
+ j8_cqlsh-dtests-py2-with-vnodes:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: medium
@@@ -444,131 -528,162 +456,314 @@@
which java
java -version
- run:
- name: Repeatedly run new or modifed JUnit tests
- no_output_timeout: 15m
+ name: Clone Cassandra dtest Repository (via git)
command: |
- set -x
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- if [ -d ~/dtest_jars ]; then
- cp ~/dtest_jars/dtest* /tmp/cassandra/build/
- fi
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Run repeated Python dtest
+ no_output_timeout: 15m
+ command: |
+ if [ "${REPEATED_DTESTS}" == "<nil>" ]; then
+ echo "Repeated dtest name hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_DTESTS_COUNT}" == "<nil>" ]; then
+ echo "Repeated dtest count hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_DTESTS_COUNT}" -le 0 ]; then
+ echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
+ else
- # Calculate the number of test iterations to be run by the current parallel runner.
- count=$((${REPEATED_UTESTS_LONG_COUNT} / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_LONG_COUNT} % CIRCLE_NODE_TOTAL))); then
- count=$((count+1))
- fi
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((${REPEATED_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
- # Put manually specified tests and automatically detected tests together, removing duplicates
- tests=$(echo ${REPEATED_UTESTS_LONG} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
- echo "Tests to be repeated: ${tests}"
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running ${REPEATED_DTESTS} $count times"
- # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
- target=long-testsome
- testtag=""
- if [[ $target == "test-cdc" ]]; then
- testtag="cdc"
- elif [[ $target == "test-compression" ]]; then
- testtag="compression"
- fi
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
- # Run each test class as many times as requested.
- exit_code="$?"
- for test in $tests; do
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
- # Split class and method names from the test name
- if [[ $test =~ "#" ]]; then
- class=${test%"#"*}
- method=${test#*"#"}
- else
- class=$test
- method=""
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+
+ tests_arg=$(echo ${REPEATED_DTESTS} | sed -e "s/,/ /g")
+
+ stop_on_failure_arg=""
+ if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
+ stop_on_failure_arg="-x"
fi
- # Prepare the -Dtest.name argument.
- # It can be the fully qualified class name or the short class name, depending on the target.
- if [[ $target == "test" || \
- $target == "test-cdc" || \
- $target == "test-compression" || \
- $target == "long-test" || \
- $target == "stress-test" ]]; then
- name_arg="-Dtest.name=${class##*.}"
- else
- name_arg="-Dtest.name=$class"
+ vnodes_args=""
+ if true; then
+ vnodes_args="--use-vnodes --num-tokens=16"
fi
- # Prepare the -Dtest.methods argument, which is optional
- if [[ $method == "" ]]; then
- methods_arg=""
- else
- methods_arg="-Dtest.methods=$method"
+ upgrade_arg=""
+ if false; then
+ upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
fi
- for i in $(seq -w 1 $count); do
- echo "Running test $test, iteration $i of $count"
-
- # run the test
- status="passes"
- if !( set -o pipefail && \
- ant long-testsome $name_arg $methods_arg -Dno-build-test=true | \
- tee stdout.txt \
- ); then
- status="fails"
- exit_code=1
- fi
-
- # move the stdout output file
- dest=/tmp/results/repeated_utests/stdout/${status}/${i}
- mkdir -p $dest
- mv stdout.txt $dest/${test}.txt
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
+ fi
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
- utests_stress_repeat:
++ j11_utests_system_keyspace_directory:
++ docker:
++ - image: apache/cassandra-testing-ubuntu2004-java11:latest
++ resource_class: medium
++ working_directory: ~/
++ shell: /bin/bash -eo pipefail -l
++ parallelism: 4
++ steps:
++ - attach_workspace:
++ at: /home/cassandra
++ - run:
++ name: Determine unit Tests to Run
++ command: |
++ # reminder: this code (along with all the steps) is independently executed on every circle container
++ # so the goal here is to get the circleci script to return the tests *this* container will run
++ # which we do via the `circleci` cli tool.
+
- # move the XML output files
- source=build/test/output/${testtag}
- dest=/tmp/results/repeated_utests/output/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
++ rm -fr ~/cassandra-dtest/upgrade_tests
++ echo "***java tests***"
+
- # move the log files
- source=build/test/logs/${testtag}
- dest=/tmp/results/repeated_utests/logs/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
++ # get all of our unit test filenames
++ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
- # maybe stop iterations on test failure
- if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
- break
- fi
- done
- done
- (exit ${exit_code})
++ # split up the unit tests into groups based on the number of containers we have
++ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
++ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
++ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
++ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
++ no_output_timeout: 15m
++ - run:
++ name: Log Environment Information
++ command: |
++ echo '*** id ***'
++ id
++ echo '*** cat /proc/cpuinfo ***'
++ cat /proc/cpuinfo
++ echo '*** free -m ***'
++ free -m
++ echo '*** df -m ***'
++ df -m
++ echo '*** ifconfig -a ***'
++ ifconfig -a
++ echo '*** uname -a ***'
++ uname -a
++ echo '*** mount ***'
++ mount
++ echo '*** env ***'
++ env
++ echo '*** java ***'
++ which java
++ java -version
++ - run:
++ name: Run Unit Tests (testclasslist-system-keyspace-directory)
++ command: |
++ set -x
++ export PATH=$JAVA_HOME/bin:$PATH
++ time mv ~/cassandra /tmp
++ cd /tmp/cassandra
++ if [ -d ~/dtest_jars ]; then
++ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
++ fi
++ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
++ if [ -z "$test_timeout" ]; then
++ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
++ fi
++ ant testclasslist-system-keyspace-directory -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
++ no_output_timeout: 15m
+ - store_test_results:
- path: /tmp/results/repeated_utests/output
- - store_artifacts:
- path: /tmp/results/repeated_utests/stdout
- destination: stdout
++ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
- path: /tmp/results/repeated_utests/output
++ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
- path: /tmp/results/repeated_utests/logs
++ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
++ - ANT_HOME: /usr/share/ant
++ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ - LANG: en_US.UTF-8
++ - KEEP_TEST_DIR: true
++ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
++ - PYTHONIOENCODING: utf-8
++ - PYTHONUNBUFFERED: true
++ - CASS_DRIVER_NO_EXTENSIONS: true
++ - CASS_DRIVER_NO_CYTHON: true
++ - CASSANDRA_SKIP_SYNC: true
++ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
++ - DTEST_BRANCH: trunk
++ - CCM_MAX_HEAP_SIZE: 1024M
++ - CCM_HEAP_NEWSIZE: 256M
++ - REPEATED_TESTS_STOP_ON_FAILURE: false
++ - REPEATED_UTESTS: null
++ - REPEATED_UTESTS_COUNT: 500
++ - REPEATED_UTESTS_FQLTOOL: null
++ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
++ - REPEATED_UTESTS_LONG: null
++ - REPEATED_UTESTS_LONG_COUNT: 100
++ - REPEATED_UTESTS_STRESS: null
++ - REPEATED_UTESTS_STRESS_COUNT: 500
++ - REPEATED_JVM_DTESTS: null
++ - REPEATED_JVM_DTESTS_COUNT: 500
++ - REPEATED_JVM_UPGRADE_DTESTS: null
++ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
++ - REPEATED_DTESTS: null
++ - REPEATED_DTESTS_COUNT: 500
++ - REPEATED_UPGRADE_DTESTS: null
++ - REPEATED_UPGRADE_DTESTS_COUNT: 25
++ - REPEATED_ANT_TEST_TARGET: testsome
++ - REPEATED_ANT_TEST_CLASS: null
++ - REPEATED_ANT_TEST_METHODS: null
++ - REPEATED_ANT_TEST_COUNT: 500
++ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - CASSANDRA_USE_JDK11: true
++ j8_utests_stress:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
++ parallelism: 1
++ steps:
++ - attach_workspace:
++ at: /home/cassandra
++ - run:
++ name: Run Unit Tests (stress-test)
++ command: |
++ export PATH=$JAVA_HOME/bin:$PATH
++ time mv ~/cassandra /tmp
++ cd /tmp/cassandra
++ if [ -d ~/dtest_jars ]; then
++ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
++ fi
++ ant stress-test
++ no_output_timeout: 15m
++ - store_test_results:
++ path: /tmp/cassandra/build/test/output/
++ - store_artifacts:
++ path: /tmp/cassandra/build/test/output
++ destination: junitxml
++ - store_artifacts:
++ path: /tmp/cassandra/build/test/logs
++ destination: logs
++ environment:
+ - ANT_HOME: /usr/share/ant
++ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
++ - REPEATED_UTESTS_FQLTOOL: null
++ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_unit_tests_repeat:
++ j11_utests_stress_repeat:
+ docker:
- - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
++ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
parallelism: 4
steps:
- attach_workspace:
@@@ -614,9 -729,18 +809,20 @@@
fi
# Put manually specified tests and automatically detected tests together, removing duplicates
- tests=$(echo ${REPEATED_UTESTS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
+ tests=$(echo ${REPEATED_UTESTS_STRESS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
echo "Tests to be repeated: ${tests}"
+ # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
- target=testsome
++ target=stress-test-some
+ testtag=""
+ if [[ $target == "test-cdc" ]]; then
+ testtag="cdc"
+ elif [[ $target == "test-compression" ]]; then
+ testtag="compression"
++ elif [[ $target == "test-system-keyspace-directory" ]]; then
++ testtag="system_keyspace_directory"
+ fi
+
# Run each test class as many times as requested.
exit_code="$?"
for test in $tests; do
@@@ -740,11 -859,11 +946,12 @@@
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
- REPEATED_ANT_TEST_COUNT: 500
-- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
-- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j11_unit_tests:
- j8_upgrade_dtests_repeat:
++ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - CASSANDRA_USE_JDK11: true
++ j8_utests_compression_repeat:
docker:
- - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
@@@ -752,75 -871,188 +959,265 @@@
steps:
- attach_workspace:
at: /home/cassandra
- - run:
- name: Determine unit Tests to Run
- command: |
- # reminder: this code (along with all the steps) is independently executed on every circle container
- # so the goal here is to get the circleci script to return the tests *this* container will run
- # which we do via the `circleci` cli tool.
-
- rm -fr ~/cassandra-dtest/upgrade_tests
- echo "***java tests***"
-
- # get all of our unit test filenames
- set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
-
- # split up the unit tests into groups based on the number of containers we have
- set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
- set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
- echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
- cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
- no_output_timeout: 15m
- run:
- name: Clone Cassandra dtest Repository (via git)
- command: |
- git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- - run:
- name: Configure virtualenv and python Dependencies
+ name: Log Environment Information
command: |
- # note, this should be super quick as all dependencies should be pre-installed in the docker image
- # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
- # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
- # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
- source ~/env3.6/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
- pip3 freeze
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
- run:
- name: Run Unit Tests (testclasslist)
- name: Run repeated Python dtest
++ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
command: |
- if [ "${REPEATED_UPGRADE_DTESTS}" == "<nil>" ]; then
- echo "Repeated dtest name hasn't been defined, exiting without running any test"
- elif [ "${REPEATED_UPGRADE_DTESTS_COUNT}" == "<nil>" ]; then
- echo "Repeated dtest count hasn't been defined, exiting without running any test"
- elif [ "${REPEATED_UPGRADE_DTESTS_COUNT}" -le 0 ]; then
- echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
- else
-
- # Calculate the number of test iterations to be run by the current parallel runner.
- # Since we are running the same test multiple times there is no need to use `circleci tests split`.
- count=$((${REPEATED_UPGRADE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (${REPEATED_UPGRADE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
- count=$((count+1))
- fi
-
- if (($count <= 0)); then
- echo "No tests to run in this runner"
- else
- echo "Running ${REPEATED_UPGRADE_DTESTS} $count times"
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
- test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
- if [ -z "$test_timeout" ]; then
- test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+
- source ~/env3.6/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
++ # Calculate the number of test iterations to be run by the current parallel runner.
++ count=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))
++ if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
++ count=$((count+1))
+ fi
- ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
- no_output_timeout: 15m
- - store_test_results:
- path: /tmp/cassandra/build/test/output/
- - store_artifacts:
- path: /tmp/cassandra/build/test/output
- destination: junitxml
- - store_artifacts:
- path: /tmp/cassandra/build/test/logs
- destination: logs
- environment:
- - ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+
- java -version
- cd ~/cassandra-dtest
- mkdir -p /tmp/dtest
++ # Put manually specified tests and automatically detected tests together, removing duplicates
++ tests=$(echo ${REPEATED_UTESTS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
++ echo "Tests to be repeated: ${tests}"
+
- echo "env: $(env)"
- echo "** done env"
- mkdir -p /tmp/results/dtests
++ # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
++ target=test-compression
++ testtag=""
++ if [[ $target == "test-cdc" ]]; then
++ testtag="cdc"
++ elif [[ $target == "test-compression" ]]; then
++ testtag="compression"
++ elif [[ $target == "test-system-keyspace-directory" ]]; then
++ testtag="system_keyspace_directory"
++ fi
+
- tests_arg=$(echo ${REPEATED_UPGRADE_DTESTS} | sed -e "s/,/ /g")
++ # Run each test class as many times as requested.
++ exit_code="$?"
++ for test in $tests; do
+
- stop_on_failure_arg=""
- if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
- stop_on_failure_arg="-x"
++ # Split class and method names from the test name
++ if [[ $test =~ "#" ]]; then
++ class=${test%"#"*}
++ method=${test#*"#"}
++ else
++ class=$test
++ method=""
+ fi
+
- vnodes_args=""
- if false; then
- vnodes_args="--use-vnodes --num-tokens=16"
++ # Prepare the -Dtest.name argument.
++ # It can be the fully qualified class name or the short class name, depending on the target.
++ if [[ $target == "test" || \
++ $target == "test-cdc" || \
++ $target == "test-compression" || \
++ $target == "test-system-keyspace-directory" || \
++ $target == "fqltool-test" || \
++ $target == "long-test" || \
++ $target == "stress-test" ]]; then
++ name_arg="-Dtest.name=${class##*.}"
++ else
++ name_arg="-Dtest.name=$class"
+ fi
+
- upgrade_arg=""
- if true; then
- upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
++ # Prepare the -Dtest.methods argument, which is optional
++ if [[ $method == "" ]]; then
++ methods_arg=""
++ else
++ methods_arg="-Dtest.methods=$method"
+ fi
+
- # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
- set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
- fi
- fi
++ for i in $(seq -w 1 $count); do
++ echo "Running test $test, iteration $i of $count"
++
++ # run the test
++ status="passes"
++ if !( set -o pipefail && \
++ ant test-compression $name_arg $methods_arg -Dno-build-test=true | \
++ tee stdout.txt \
++ ); then
++ status="fails"
++ exit_code=1
++ fi
++
++ # move the stdout output file
++ dest=/tmp/results/repeated_utests/stdout/${status}/${i}
++ mkdir -p $dest
++ mv stdout.txt $dest/${test}.txt
++
++ # move the XML output files
++ source=build/test/output/${testtag}
++ dest=/tmp/results/repeated_utests/output/${status}/${i}
++ mkdir -p $dest
++ if [[ -d $source && -n "$(ls $source)" ]]; then
++ mv $source/* $dest/
++ fi
++
++ # move the log files
++ source=build/test/logs/${testtag}
++ dest=/tmp/results/repeated_utests/logs/${status}/${i}
++ mkdir -p $dest
++ if [[ -d $source && -n "$(ls $source)" ]]; then
++ mv $source/* $dest/
++ fi
++
++ # maybe stop iterations on test failure
++ if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
++ break
++ fi
++ done
++ done
++ (exit ${exit_code})
+ - store_test_results:
- path: /tmp/results
++ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
- path: /tmp/dtest
- destination: dtest
++ path: /tmp/results/repeated_utests/stdout
++ destination: stdout
+ - store_artifacts:
- path: ~/cassandra-dtest/logs
- destination: dtest_logs
++ path: /tmp/results/repeated_utests/output
++ destination: junitxml
++ - store_artifacts:
++ path: /tmp/results/repeated_utests/logs
++ destination: logs
+ environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - ANT_HOME: /usr/share/ant
++ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
++ - REPEATED_UTESTS_FQLTOOL: null
++ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- build:
++ j11_unit_tests:
+ docker:
- - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
++ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
- parallelism: 1
++ parallelism: 4
+ steps:
++ - attach_workspace:
++ at: /home/cassandra
++ - run:
++ name: Determine unit Tests to Run
++ command: |
++ # reminder: this code (along with all the steps) is independently executed on every circle container
++ # so the goal here is to get the circleci script to return the tests *this* container will run
++ # which we do via the `circleci` cli tool.
++
++ rm -fr ~/cassandra-dtest/upgrade_tests
++ echo "***java tests***"
++
++ # get all of our unit test filenames
++ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
++
++ # split up the unit tests into groups based on the number of containers we have
++ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
++ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
++ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
++ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
++ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
- name: Clone Cassandra Repository (via git)
- command: |
- git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH https://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra
- - run:
- name: Build Cassandra
++ name: Run Unit Tests (testclasslist)
+ command: |
++ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
- cd ~/cassandra
- # Loop to prevent failure due to maven-ant-tasks not downloading a jar..
- for x in $(seq 1 3); do
- ${ANT_HOME}/bin/ant clean jar build-test
- RETURN="$?"
- if [ "${RETURN}" -eq "0" ]; then
- break
- fi
- done
- # Exit, if we didn't build successfully
- if [ "${RETURN}" -ne "0" ]; then
- echo "Build failed with exit code: ${RETURN}"
- exit ${RETURN}
++ time mv ~/cassandra /tmp
++ cd /tmp/cassandra
++ if [ -d ~/dtest_jars ]; then
++ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
++ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
++ if [ -z "$test_timeout" ]; then
++ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
++ fi
++ ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
+ no_output_timeout: 15m
- - run:
- name: Run eclipse-warnings
- command: |
- export PATH=$JAVA_HOME/bin:$PATH
- cd ~/cassandra
- ant eclipse-warnings
- - persist_to_workspace:
- root: /home/cassandra
- paths:
- - cassandra
- - .m2
++ - store_test_results:
++ path: /tmp/cassandra/build/test/output/
++ - store_artifacts:
++ path: /tmp/cassandra/build/test/output
++ destination: junitxml
++ - store_artifacts:
++ path: /tmp/cassandra/build/test/logs
++ destination: logs
+ environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - ANT_HOME: /usr/share/ant
++ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@@ -880,3185 -1109,75 +1277,4942 @@@
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
pip3 freeze
- run:
- name: Run repeated Python dtest
+ name: Determine Tests to Run (j8_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-i [...]
+ - run:
+ name: Run dtests (j8_without_vnodes)
no_output_timeout: 15m
command: |
- if [ "${REPEATED_DTESTS}" == "<nil>" ]; then
- echo "Repeated dtest name hasn't been defined, exiting without running any test"
- elif [ "${REPEATED_DTESTS_COUNT}" == "<nil>" ]; then
- echo "Repeated dtest count hasn't been defined, exiting without running any test"
- elif [ "${REPEATED_DTESTS_COUNT}" -le 0 ]; then
- echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
+ echo "cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.6
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j8_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j8_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j11_cqlsh_dtests_py38:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource- [...]
+ - run:
+ name: Run dtests (j11_without_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt
+
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.8
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
- j11_repeated_ant_test:
++ j11_utests_compression_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
- name: Run repeated JUnit test
++ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: |
- if [ "${REPEATED_ANT_TEST_CLASS}" == "<nil>" ]; then
- echo "Repeated utest class name hasn't been defined, exiting without running any test"
- elif [ "${REPEATED_ANT_TEST_COUNT}" == "<nil>" ]; then
- echo "Repeated utest count hasn't been defined, exiting without running any test"
- elif [ "${REPEATED_ANT_TEST_COUNT}" -le 0 ]; then
- echo "Repeated utest count is lesser or equals than zero, exiting without running any test"
- else
-
- # Calculate the number of test iterations to be run by the current parallel runner.
- # Since we are running the same test multiple times there is no need to use `circleci tests split`.
- count=$((${REPEATED_ANT_TEST_COUNT} / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (${REPEATED_ANT_TEST_COUNT} % CIRCLE_NODE_TOTAL))); then
- count=$((count+1))
- fi
++ set -x
++ export PATH=$JAVA_HOME/bin:$PATH
++ time mv ~/cassandra /tmp
++ cd /tmp/cassandra
++ if [ -d ~/dtest_jars ]; then
++ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
++ fi
+
- if (($count <= 0)); then
- echo "No tests to run in this runner"
- else
- echo "Running ${REPEATED_ANT_TEST_TARGET} ${REPEATED_ANT_TEST_CLASS} ${REPEATED_ANT_TEST_METHODS} ${REPEATED_ANT_TEST_COUNT} times"
++ # Calculate the number of test iterations to be run by the current parallel runner.
++ count=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))
++ if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
++ count=$((count+1))
++ fi
+
- set -x
- export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- if [ -d ~/dtest_jars ]; then
- cp ~/dtest_jars/dtest* /tmp/cassandra/build/
- fi
++ # Put manually specified tests and automatically detected tests together, removing duplicates
++ tests=$(echo ${REPEATED_UTESTS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
++ echo "Tests to be repeated: ${tests}"
+
- target=${REPEATED_ANT_TEST_TARGET}
- class_path=${REPEATED_ANT_TEST_CLASS}
- class_name="${class_path##*.}"
++ # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
++ target=test-compression
++ testtag=""
++ if [[ $target == "test-cdc" ]]; then
++ testtag="cdc"
++ elif [[ $target == "test-compression" ]]; then
++ testtag="compression"
++ elif [[ $target == "test-system-keyspace-directory" ]]; then
++ testtag="system_keyspace_directory"
++ fi
+
- # Prepare the -Dtest.name argument.
++ # Run each test class as many times as requested.
++ exit_code="$?"
++ for test in $tests; do
++
++ # Split class and method names from the test name
++ if [[ $test =~ "#" ]]; then
++ class=${test%"#"*}
++ method=${test#*"#"}
++ else
++ class=$test
++ method=""
++ fi
++
++ # Prepare the -Dtest.name argument.
+ # It can be the fully qualified class name or the short class name, depending on the target.
+ if [[ $target == "test" || \
+ $target == "test-cdc" || \
+ $target == "test-compression" || \
+ $target == "test-system-keyspace-directory" || \
+ $target == "fqltool-test" || \
+ $target == "long-test" || \
+ $target == "stress-test" ]]; then
- name="-Dtest.name=$class_name"
++ name_arg="-Dtest.name=${class##*.}"
+ else
- name="-Dtest.name=$class_path"
++ name_arg="-Dtest.name=$class"
+ fi
+
+ # Prepare the -Dtest.methods argument, which is optional
- if [ "${REPEATED_ANT_TEST_METHODS}" == "<nil>" ]; then
- methods=""
++ if [[ $method == "" ]]; then
++ methods_arg=""
+ else
- methods="-Dtest.methods=${REPEATED_ANT_TEST_METHODS}"
++ methods_arg="-Dtest.methods=$method"
+ fi
+
- # Run the test target as many times as requested collecting the exit code,
- # stopping the iteration only if stop_on_failure is set.
- exit_code="$?"
+ for i in $(seq -w 1 $count); do
-
- echo "Running test iteration $i of $count"
++ echo "Running test $test, iteration $i of $count"
+
+ # run the test
+ status="passes"
- if !( set -o pipefail && ant $target $name $methods -Dno-build-test=true | tee stdout.txt ); then
++ if !( set -o pipefail && \
++ ant test-compression $name_arg $methods_arg -Dno-build-test=true | \
++ tee stdout.txt \
++ ); then
+ status="fails"
+ exit_code=1
+ fi
+
+ # move the stdout output file
- dest=/tmp/results/repeated_utest/stdout/${status}/${i}
++ dest=/tmp/results/repeated_utests/stdout/${status}/${i}
+ mkdir -p $dest
- mv stdout.txt $dest/${REPEATED_ANT_TEST_TARGET}-${REPEATED_ANT_TEST_CLASS}.txt
++ mv stdout.txt $dest/${test}.txt
+
+ # move the XML output files
- source=build/test/output
- dest=/tmp/results/repeated_utest/output/${status}/${i}
++ source=build/test/output/${testtag}
++ dest=/tmp/results/repeated_utests/output/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # move the log files
- source=build/test/logs
- dest=/tmp/results/repeated_utest/logs/${status}/${i}
++ source=build/test/logs/${testtag}
++ dest=/tmp/results/repeated_utests/logs/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # maybe stop iterations on test failure
+ if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
+ break
+ fi
+ done
-
- (exit ${exit_code})
- fi
- fi
++ done
++ (exit ${exit_code})
+ - store_test_results:
- path: /tmp/results/repeated_utest/output
++ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
- path: /tmp/results/repeated_utest/stdout
++ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
- path: /tmp/results/repeated_utest/output
++ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
- path: /tmp/results/repeated_utest/logs
++ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
- j8_cqlsh_dtests_py3_vnode:
++ j11_repeated_ant_test:
+ docker:
- - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
++ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
- name: Clone Cassandra dtest Repository (via git)
- command: |
- git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- - run:
- name: Configure virtualenv and python Dependencies
++ name: Log Environment Information
+ command: |
- # note, this should be super quick as all dependencies should be pre-installed in the docker image
- # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
- # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
- # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
- source ~/env3.6/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
- pip3 uninstall -y cqlsh
- pip3 freeze
- - run:
- name: Determine Tests to Run (j8_with_vnodes)
- no_output_timeout: 5m
- command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip- [...]
++ echo '*** id ***'
++ id
++ echo '*** cat /proc/cpuinfo ***'
++ cat /proc/cpuinfo
++ echo '*** free -m ***'
++ free -m
++ echo '*** df -m ***'
++ df -m
++ echo '*** ifconfig -a ***'
++ ifconfig -a
++ echo '*** uname -a ***'
++ uname -a
++ echo '*** mount ***'
++ mount
++ echo '*** env ***'
++ env
++ echo '*** java ***'
++ which java
++ java -version
+ - run:
- name: Run dtests (j8_with_vnodes)
++ name: Run repeated JUnit test
+ no_output_timeout: 15m
+ command: |
- echo "cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt"
- cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt
++ if [ "${REPEATED_ANT_TEST_CLASS}" == "<nil>" ]; then
++ echo "Repeated utest class name hasn't been defined, exiting without running any test"
++ elif [ "${REPEATED_ANT_TEST_COUNT}" == "<nil>" ]; then
++ echo "Repeated utest count hasn't been defined, exiting without running any test"
++ elif [ "${REPEATED_ANT_TEST_COUNT}" -le 0 ]; then
++ echo "Repeated utest count is lesser or equals than zero, exiting without running any test"
++ else
+
- source ~/env3.6/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
- if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then
- export CQLSH_PYTHON=/usr/bin/python3.6
- fi
++ # Calculate the number of test iterations to be run by the current parallel runner.
++ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
++ count=$((${REPEATED_ANT_TEST_COUNT} / CIRCLE_NODE_TOTAL))
++ if (($CIRCLE_NODE_INDEX < (${REPEATED_ANT_TEST_COUNT} % CIRCLE_NODE_TOTAL))); then
++ count=$((count+1))
++ fi
+
- java -version
- cd ~/cassandra-dtest
- mkdir -p /tmp/dtest
++ if (($count <= 0)); then
++ echo "No tests to run in this runner"
++ else
++ echo "Running ${REPEATED_ANT_TEST_TARGET} ${REPEATED_ANT_TEST_CLASS} ${REPEATED_ANT_TEST_METHODS} ${REPEATED_ANT_TEST_COUNT} times"
+
- echo "env: $(env)"
- echo "** done env"
- mkdir -p /tmp/results/dtests
- # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
- export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`
- set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
++ set -x
++ export PATH=$JAVA_HOME/bin:$PATH
++ time mv ~/cassandra /tmp
++ cd /tmp/cassandra
++ if [ -d ~/dtest_jars ]; then
++ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
++ fi
++
++ target=${REPEATED_ANT_TEST_TARGET}
++ class_path=${REPEATED_ANT_TEST_CLASS}
++ class_name="${class_path##*.}"
++
++ # Prepare the -Dtest.name argument.
++ # It can be the fully qualified class name or the short class name, depending on the target.
++ if [[ $target == "test" || \
++ $target == "test-cdc" || \
++ $target == "test-compression" || \
++ $target == "test-system-keyspace-directory" || \
++ $target == "fqltool-test" || \
++ $target == "long-test" || \
++ $target == "stress-test" ]]; then
++ name="-Dtest.name=$class_name"
++ else
++ name="-Dtest.name=$class_path"
++ fi
++
++ # Prepare the -Dtest.methods argument, which is optional
++ if [ "${REPEATED_ANT_TEST_METHODS}" == "<nil>" ]; then
++ methods=""
++ else
++ methods="-Dtest.methods=${REPEATED_ANT_TEST_METHODS}"
++ fi
++
++ # Run the test target as many times as requested collecting the exit code,
++ # stopping the iteration only if stop_on_failure is set.
++ exit_code="$?"
++ for i in $(seq -w 1 $count); do
++
++ echo "Running test iteration $i of $count"
++
++ # run the test
++ status="passes"
++ if !( set -o pipefail && ant $target $name $methods -Dno-build-test=true | tee stdout.txt ); then
++ status="fails"
++ exit_code=1
++ fi
++
++ # move the stdout output file
++ dest=/tmp/results/repeated_utest/stdout/${status}/${i}
++ mkdir -p $dest
++ mv stdout.txt $dest/${REPEATED_ANT_TEST_TARGET}-${REPEATED_ANT_TEST_CLASS}.txt
++
++ # move the XML output files
++ source=build/test/output
++ dest=/tmp/results/repeated_utest/output/${status}/${i}
++ mkdir -p $dest
++ if [[ -d $source && -n "$(ls $source)" ]]; then
++ mv $source/* $dest/
++ fi
++
++ # move the log files
++ source=build/test/logs
++ dest=/tmp/results/repeated_utest/logs/${status}/${i}
++ mkdir -p $dest
++ if [[ -d $source && -n "$(ls $source)" ]]; then
++ mv $source/* $dest/
++ fi
++
++ # maybe stop iterations on test failure
++ if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
++ break
++ fi
++ done
++
++ (exit ${exit_code})
++ fi
++ fi
+ - store_test_results:
- path: /tmp/results
++ path: /tmp/results/repeated_utest/output
+ - store_artifacts:
- path: /tmp/dtest
- destination: dtest_j8_with_vnodes
++ path: /tmp/results/repeated_utest/stdout
++ destination: stdout
+ - store_artifacts:
- path: ~/cassandra-dtest/logs
- destination: dtest_j8_with_vnodes_logs
++ path: /tmp/results/repeated_utest/output
++ destination: junitxml
++ - store_artifacts:
++ path: /tmp/results/repeated_utest/logs
++ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j11_cqlsh_dtests_py3:
++ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - CASSANDRA_USE_JDK11: true
++ j11_utests_system_keyspace_directory_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
- name: Clone Cassandra dtest Repository (via git)
++ name: Log Environment Information
+ command: |
- git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
++ echo '*** id ***'
++ id
++ echo '*** cat /proc/cpuinfo ***'
++ cat /proc/cpuinfo
++ echo '*** free -m ***'
++ free -m
++ echo '*** df -m ***'
++ df -m
++ echo '*** ifconfig -a ***'
++ ifconfig -a
++ echo '*** uname -a ***'
++ uname -a
++ echo '*** mount ***'
++ mount
++ echo '*** env ***'
++ env
++ echo '*** java ***'
++ which java
++ java -version
+ - run:
- name: Configure virtualenv and python Dependencies
- command: |
- # note, this should be super quick as all dependencies should be pre-installed in the docker image
- # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
- # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
- # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
- source ~/env3.6/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
- pip3 uninstall -y cqlsh
- pip3 freeze
- - run:
- name: Determine Tests to Run (j11_without_vnodes)
- no_output_timeout: 5m
- command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource- [...]
- - run:
- name: Run dtests (j11_without_vnodes)
- no_output_timeout: 15m
- command: |
- echo "cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt"
- cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt
-
- source ~/env3.6/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
- if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then
- export CQLSH_PYTHON=/usr/bin/python3.6
- fi
-
- java -version
- cd ~/cassandra-dtest
- mkdir -p /tmp/dtest
-
- echo "env: $(env)"
- echo "** done env"
- mkdir -p /tmp/results/dtests
- # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
- export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt`
- set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
- - store_test_results:
- path: /tmp/results
- - store_artifacts:
- path: /tmp/dtest
- destination: dtest_j11_without_vnodes
- - store_artifacts:
- path: ~/cassandra-dtest/logs
- destination: dtest_j11_without_vnodes_logs
- environment:
- - ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - LANG: en_US.UTF-8
- - KEEP_TEST_DIR: true
- - DEFAULT_DIR: /home/cassandra/cassandra-dtest
- - PYTHONIOENCODING: utf-8
- - PYTHONUNBUFFERED: true
- - CASS_DRIVER_NO_EXTENSIONS: true
- - CASS_DRIVER_NO_CYTHON: true
- - CASSANDRA_SKIP_SYNC: true
- - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: trunk
- - CCM_MAX_HEAP_SIZE: 1024M
- - CCM_HEAP_NEWSIZE: 256M
- - REPEATED_TESTS_STOP_ON_FAILURE: false
- - REPEATED_UTESTS: null
- - REPEATED_UTESTS_COUNT: 500
- - REPEATED_UTESTS_FQLTOOL: null
- - REPEATED_UTESTS_FQLTOOL_COUNT: 500
- - REPEATED_UTESTS_LONG: null
- - REPEATED_UTESTS_LONG_COUNT: 100
- - REPEATED_UTESTS_STRESS: null
- - REPEATED_UTESTS_STRESS_COUNT: 500
- - REPEATED_JVM_DTESTS: null
- - REPEATED_JVM_DTESTS_COUNT: 500
- - REPEATED_JVM_UPGRADE_DTESTS: null
- - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- - REPEATED_DTESTS: null
- - REPEATED_DTESTS_COUNT: 500
- - REPEATED_UPGRADE_DTESTS: null
- - REPEATED_UPGRADE_DTESTS_COUNT: 25
- - REPEATED_ANT_TEST_TARGET: testsome
- - REPEATED_ANT_TEST_CLASS: null
- - REPEATED_ANT_TEST_METHODS: null
- - REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - CASSANDRA_USE_JDK11: true
- utests_long_repeat:
- docker:
- - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
- resource_class: medium
- working_directory: ~/
- shell: /bin/bash -eo pipefail -l
- parallelism: 4
- steps:
- - attach_workspace:
- at: /home/cassandra
- - run:
- name: Log Environment Information
- command: |
- echo '*** id ***'
- id
- echo '*** cat /proc/cpuinfo ***'
- cat /proc/cpuinfo
- echo '*** free -m ***'
- free -m
- echo '*** df -m ***'
- df -m
- echo '*** ifconfig -a ***'
- ifconfig -a
- echo '*** uname -a ***'
- uname -a
- echo '*** mount ***'
- mount
- echo '*** env ***'
- env
- echo '*** java ***'
- which java
- java -version
- - run:
- name: Repeatedly run new or modifed JUnit tests
- no_output_timeout: 15m
++ name: Repeatedly run new or modifed JUnit tests
++ no_output_timeout: 15m
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
- count=$((${REPEATED_UTESTS_LONG_COUNT} / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_LONG_COUNT} % CIRCLE_NODE_TOTAL))); then
++ count=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))
++ if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ # Put manually specified tests and automatically detected tests together, removing duplicates
- tests=$(echo ${REPEATED_UTESTS_LONG} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
++ tests=$(echo ${REPEATED_UTESTS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
+ echo "Tests to be repeated: ${tests}"
+
++ # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
++ target=test-system-keyspace-directory
++ testtag=""
++ if [[ $target == "test-cdc" ]]; then
++ testtag="cdc"
++ elif [[ $target == "test-compression" ]]; then
++ testtag="compression"
++ elif [[ $target == "test-system-keyspace-directory" ]]; then
++ testtag="system_keyspace_directory"
++ fi
++
+ # Run each test class as many times as requested.
+ exit_code="$?"
+ for test in $tests; do
+
+ # Split class and method names from the test name
+ if [[ $test =~ "#" ]]; then
+ class=${test%"#"*}
+ method=${test#*"#"}
+ else
+ class=$test
+ method=""
+ fi
+
+ # Prepare the -Dtest.name argument.
+ # It can be the fully qualified class name or the short class name, depending on the target.
+ if [[ $target == "test" || \
+ $target == "test-cdc" || \
+ $target == "test-compression" || \
+ $target == "test-system-keyspace-directory" || \
+ $target == "fqltool-test" || \
+ $target == "long-test" || \
+ $target == "stress-test" ]]; then
+ name_arg="-Dtest.name=${class##*.}"
+ else
+ name_arg="-Dtest.name=$class"
+ fi
+
+ # Prepare the -Dtest.methods argument, which is optional
- if [ $method == "" ]; then
++ if [[ $method == "" ]]; then
+ methods_arg=""
+ else
+ methods_arg="-Dtest.methods=$method"
+ fi
+
+ for i in $(seq -w 1 $count); do
+ echo "Running test $test, iteration $i of $count"
+
+ # run the test
+ status="passes"
+ if !( set -o pipefail && \
- ant long-testsome $name_arg $methods_arg -Dno-build-test=true | \
++ ant test-system-keyspace-directory $name_arg $methods_arg -Dno-build-test=true | \
+ tee stdout.txt \
+ ); then
+ status="fails"
+ exit_code=1
+ fi
+
+ # move the stdout output file
+ dest=/tmp/results/repeated_utests/stdout/${status}/${i}
+ mkdir -p $dest
+ mv stdout.txt $dest/${test}.txt
+
+ # move the XML output files
- source=build/test/output
++ source=build/test/output/${testtag}
+ dest=/tmp/results/repeated_utests/output/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # move the log files
- source=build/test/logs
++ source=build/test/logs/${testtag}
+ dest=/tmp/results/repeated_utests/logs/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # maybe stop iterations on test failure
+ if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
+ break
+ fi
+ done
+ done
+ (exit ${exit_code})
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_unit_tests_repeat:
++ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - CASSANDRA_USE_JDK11: true
++ j8_cqlsh_dtests_py3_vnode:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
- name: Log Environment Information
++ name: Clone Cassandra dtest Repository (via git)
+ command: |
- echo '*** id ***'
- id
- echo '*** cat /proc/cpuinfo ***'
- cat /proc/cpuinfo
- echo '*** free -m ***'
- free -m
- echo '*** df -m ***'
- df -m
- echo '*** ifconfig -a ***'
- ifconfig -a
- echo '*** uname -a ***'
- uname -a
- echo '*** mount ***'
- mount
- echo '*** env ***'
- env
- echo '*** java ***'
- which java
- java -version
++ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
- name: Repeatedly run new or modifed JUnit tests
- no_output_timeout: 15m
++ name: Configure virtualenv and python Dependencies
+ command: |
- set -x
++ # note, this should be super quick as all dependencies should be pre-installed in the docker image
++ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
++ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
++ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
++ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- if [ -d ~/dtest_jars ]; then
- cp ~/dtest_jars/dtest* /tmp/cassandra/build/
- fi
++ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
++ pip3 uninstall -y cqlsh
++ pip3 freeze
++ - run:
++ name: Determine Tests to Run (j8_with_vnodes)
++ no_output_timeout: 5m
++ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip- [...]
++ - run:
++ name: Run dtests (j8_with_vnodes)
++ no_output_timeout: 15m
++ command: |
++ echo "cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt"
++ cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt
+
- # Calculate the number of test iterations to be run by the current parallel runner.
- count=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
- count=$((count+1))
++ source ~/env3.6/bin/activate
++ export PATH=$JAVA_HOME/bin:$PATH
++ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then
++ export CQLSH_PYTHON=/usr/bin/python3.6
+ fi
+
- # Put manually specified tests and automatically detected tests together, removing duplicates
- tests=$(echo ${REPEATED_UTESTS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
- echo "Tests to be repeated: ${tests}"
-
- # Run each test class as many times as requested.
- exit_code="$?"
- for test in $tests; do
-
- # Split class and method names from the test name
- if [[ $test =~ "#" ]]; then
- class=${test%"#"*}
- method=${test#*"#"}
- else
- class=$test
- method=""
- fi
-
- # Prepare the -Dtest.name argument.
- # It can be the fully qualified class name or the short class name, depending on the target.
- if [[ $target == "test" || \
- $target == "test-cdc" || \
- $target == "test-compression" || \
- $target == "test-system-keyspace-directory" || \
- $target == "fqltool-test" || \
- $target == "long-test" || \
- $target == "stress-test" ]]; then
- name_arg="-Dtest.name=${class##*.}"
- else
- name_arg="-Dtest.name=$class"
- fi
-
- # Prepare the -Dtest.methods argument, which is optional
- if [ $method == "" ]; then
- methods_arg=""
- else
- methods_arg="-Dtest.methods=$method"
- fi
-
- for i in $(seq -w 1 $count); do
- echo "Running test $test, iteration $i of $count"
-
- # run the test
- status="passes"
- if !( set -o pipefail && \
- ant testsome $name_arg $methods_arg -Dno-build-test=true | \
- tee stdout.txt \
- ); then
- status="fails"
- exit_code=1
- fi
-
- # move the stdout output file
- dest=/tmp/results/repeated_utests/stdout/${status}/${i}
- mkdir -p $dest
- mv stdout.txt $dest/${test}.txt
++ java -version
++ cd ~/cassandra-dtest
++ mkdir -p /tmp/dtest
+
- # move the XML output files
- source=build/test/output
- dest=/tmp/results/repeated_utests/output/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # move the log files
- source=build/test/logs
- dest=/tmp/results/repeated_utests/logs/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # maybe stop iterations on test failure
- if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
- break
- fi
- done
- done
- (exit ${exit_code})
++ echo "env: $(env)"
++ echo "** done env"
++ mkdir -p /tmp/results/dtests
++ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
++ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`
++ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
- path: /tmp/results/repeated_utests/output
- - store_artifacts:
- path: /tmp/results/repeated_utests/stdout
- destination: stdout
++ path: /tmp/results
+ - store_artifacts:
- path: /tmp/results/repeated_utests/output
- destination: junitxml
++ path: /tmp/dtest
++ destination: dtest_j8_with_vnodes
+ - store_artifacts:
- path: /tmp/results/repeated_utests/logs
- destination: logs
++ path: ~/cassandra-dtest/logs
++ destination: dtest_j8_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_cqlsh-dtests-py2-no-vnodes:
++ j11_cqlsh_dtests_py3:
+ docker:
- - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
++ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
- name: Determine Tests to Run (j8_without_vnodes)
++ name: Determine Tests to Run (j11_without_vnodes)
+ no_output_timeout: 5m
- command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-i [...]
++ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource- [...]
+ - run:
- name: Run dtests (j8_without_vnodes)
++ name: Run dtests (j11_without_vnodes)
+ no_output_timeout: 15m
+ command: |
- echo "cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt"
- cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt
++ echo "cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt"
++ cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
- if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then
- export CQLSH_PYTHON=/usr/bin/python2.7
++ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then
++ export CQLSH_PYTHON=/usr/bin/python3.6
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
- export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`
- set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
++ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt`
++ set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
- destination: dtest_j8_without_vnodes
++ destination: dtest_j11_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
- destination: dtest_j8_without_vnodes_logs
++ destination: dtest_j11_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_upgrade_dtests_repeat:
++ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - CASSANDRA_USE_JDK11: true
++ j11_utests_cdc:
+ docker:
- - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
++ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
- name: Clone Cassandra dtest Repository (via git)
++ name: Determine unit Tests to Run
+ command: |
- git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
++ # reminder: this code (along with all the steps) is independently executed on every circle container
++ # so the goal here is to get the circleci script to return the tests *this* container will run
++ # which we do via the `circleci` cli tool.
++
++ rm -fr ~/cassandra-dtest/upgrade_tests
++ echo "***java tests***"
++
++ # get all of our unit test filenames
++ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
++
++ # split up the unit tests into groups based on the number of containers we have
++ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
++ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
++ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
++ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
++ no_output_timeout: 15m
+ - run:
- name: Configure virtualenv and python Dependencies
++ name: Log Environment Information
+ command: |
- # note, this should be super quick as all dependencies should be pre-installed in the docker image
- # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
- # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
- # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
- source ~/env3.6/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
- pip3 uninstall -y cqlsh
- pip3 freeze
++ echo '*** id ***'
++ id
++ echo '*** cat /proc/cpuinfo ***'
++ cat /proc/cpuinfo
++ echo '*** free -m ***'
++ free -m
++ echo '*** df -m ***'
++ df -m
++ echo '*** ifconfig -a ***'
++ ifconfig -a
++ echo '*** uname -a ***'
++ uname -a
++ echo '*** mount ***'
++ mount
++ echo '*** env ***'
++ env
++ echo '*** java ***'
++ which java
++ java -version
+ - run:
- name: Run repeated Python dtest
- no_output_timeout: 15m
++ name: Run Unit Tests (testclasslist-cdc)
+ command: |
- if [ "${REPEATED_UPGRADE_DTESTS}" == "<nil>" ]; then
- echo "Repeated dtest name hasn't been defined, exiting without running any test"
- elif [ "${REPEATED_UPGRADE_DTESTS_COUNT}" == "<nil>" ]; then
- echo "Repeated dtest count hasn't been defined, exiting without running any test"
- elif [ "${REPEATED_UPGRADE_DTESTS_COUNT}" -le 0 ]; then
- echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
- else
-
- # Calculate the number of test iterations to be run by the current parallel runner.
- # Since we are running the same test multiple times there is no need to use `circleci tests split`.
- count=$((${REPEATED_UPGRADE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (${REPEATED_UPGRADE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
- count=$((count+1))
- fi
-
- if (($count <= 0)); then
- echo "No tests to run in this runner"
- else
- echo "Running ${REPEATED_UPGRADE_DTESTS} $count times"
-
- source ~/env3.6/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
-
- java -version
- cd ~/cassandra-dtest
- mkdir -p /tmp/dtest
-
- echo "env: $(env)"
- echo "** done env"
- mkdir -p /tmp/results/dtests
-
- tests_arg=$(echo ${REPEATED_UPGRADE_DTESTS} | sed -e "s/,/ /g")
-
- stop_on_failure_arg=""
- if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
- stop_on_failure_arg="-x"
- fi
-
- vnodes_args=""
- if false; then
- vnodes_args="--use-vnodes --num-tokens=16"
- fi
-
- upgrade_arg=""
- if true; then
- upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
- fi
-
- # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
- set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
- fi
++ set -x
++ export PATH=$JAVA_HOME/bin:$PATH
++ time mv ~/cassandra /tmp
++ cd /tmp/cassandra
++ if [ -d ~/dtest_jars ]; then
++ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
++ fi
++ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
++ if [ -z "$test_timeout" ]; then
++ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
++ ant testclasslist-cdc -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
++ no_output_timeout: 15m
+ - store_test_results:
- path: /tmp/results
++ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
- path: /tmp/dtest
- destination: dtest
++ path: /tmp/cassandra/build/test/output
++ destination: junitxml
+ - store_artifacts:
- path: ~/cassandra-dtest/logs
- destination: dtest_logs
++ path: /tmp/cassandra/build/test/logs
++ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j11_cqlsh-dtests-py2-with-vnodes:
++ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - CASSANDRA_USE_JDK11: true
++ j11_utests_fqltool:
+ docker:
- - image: apache/cassandra-testing-ubuntu2004-java11:latest
++ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
- parallelism: 4
++ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
- name: Clone Cassandra dtest Repository (via git)
- command: |
- git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- - run:
- name: Configure virtualenv and python Dependencies
- command: |
- # note, this should be super quick as all dependencies should be pre-installed in the docker image
- # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
- # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
- # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
- source ~/env3.6/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
- pip3 uninstall -y cqlsh
- pip3 freeze
- - run:
- name: Determine Tests to Run (j11_with_vnodes)
- no_output_timeout: 5m
- command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip [...]
- - run:
- name: Run dtests (j11_with_vnodes)
- no_output_timeout: 15m
++ name: Run Unit Tests (fqltool-test)
+ command: |
- echo "cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt"
- cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt
-
- source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
- if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then
- export CQLSH_PYTHON=/usr/bin/python2.7
++ time mv ~/cassandra /tmp
++ cd /tmp/cassandra
++ if [ -d ~/dtest_jars ]; then
++ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
-
- java -version
- cd ~/cassandra-dtest
- mkdir -p /tmp/dtest
-
- echo "env: $(env)"
- echo "** done env"
- mkdir -p /tmp/results/dtests
- # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
- export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt`
- set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
++ ant fqltool-test
++ no_output_timeout: 15m
+ - store_test_results:
- path: /tmp/results
- - store_artifacts:
- path: /tmp/dtest
- destination: dtest_j11_with_vnodes
++ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
- path: ~/cassandra-dtest/logs
- destination: dtest_j11_with_vnodes_logs
++ path: /tmp/cassandra/build/test/output
++ destination: junitxml
++ - store_artifacts:
++ path: /tmp/cassandra/build/test/logs
++ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - CASSANDRA_USE_JDK11: true
- utests_system_keyspace_directory_repeat:
++ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ j8_utests_system_keyspace_directory:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
++ - run:
++ name: Determine unit Tests to Run
++ command: |
++ # reminder: this code (along with all the steps) is independently executed on every circle container
++ # so the goal here is to get the circleci script to return the tests *this* container will run
++ # which we do via the `circleci` cli tool.
++
++ rm -fr ~/cassandra-dtest/upgrade_tests
++ echo "***java tests***"
++
++ # get all of our unit test filenames
++ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
++
++ # split up the unit tests into groups based on the number of containers we have
++ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
++ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
++ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
++ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
++ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
- name: Repeatedly run new or modifed JUnit tests
- no_output_timeout: 15m
++ name: Run Unit Tests (testclasslist-system-keyspace-directory)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
-
- # Calculate the number of test iterations to be run by the current parallel runner.
- count=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
- count=$((count+1))
++ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
++ if [ -z "$test_timeout" ]; then
++ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
-
- # Put manually specified tests and automatically detected tests together, removing duplicates
- tests=$(echo ${REPEATED_UTESTS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
- echo "Tests to be repeated: ${tests}"
-
- # Run each test class as many times as requested.
- exit_code="$?"
- for test in $tests; do
-
- # Split class and method names from the test name
- if [[ $test =~ "#" ]]; then
- class=${test%"#"*}
- method=${test#*"#"}
- else
- class=$test
- method=""
- fi
-
- # Prepare the -Dtest.name argument.
- # It can be the fully qualified class name or the short class name, depending on the target.
- if [[ $target == "test" || \
- $target == "test-cdc" || \
- $target == "test-compression" || \
- $target == "test-system-keyspace-directory" || \
- $target == "fqltool-test" || \
- $target == "long-test" || \
- $target == "stress-test" ]]; then
- name_arg="-Dtest.name=${class##*.}"
- else
- name_arg="-Dtest.name=$class"
- fi
-
- # Prepare the -Dtest.methods argument, which is optional
- if [ $method == "" ]; then
- methods_arg=""
- else
- methods_arg="-Dtest.methods=$method"
- fi
-
- for i in $(seq -w 1 $count); do
- echo "Running test $test, iteration $i of $count"
-
- # run the test
- status="passes"
- if !( set -o pipefail && \
- ant test-system-keyspace-directory $name_arg $methods_arg -Dno-build-test=true | \
- tee stdout.txt \
- ); then
- status="fails"
- exit_code=1
- fi
-
- # move the stdout output file
- dest=/tmp/results/repeated_utests/stdout/${status}/${i}
- mkdir -p $dest
- mv stdout.txt $dest/${test}.txt
-
- # move the XML output files
- source=build/test/output
- dest=/tmp/results/repeated_utests/output/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # move the log files
- source=build/test/logs
- dest=/tmp/results/repeated_utests/logs/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # maybe stop iterations on test failure
- if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
- break
- fi
- done
- done
- (exit ${exit_code})
++ ant testclasslist-system-keyspace-directory -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
++ no_output_timeout: 15m
+ - store_test_results:
- path: /tmp/results/repeated_utests/output
- - store_artifacts:
- path: /tmp/results/repeated_utests/stdout
- destination: stdout
++ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
- path: /tmp/results/repeated_utests/output
++ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
- path: /tmp/results/repeated_utests/logs
++ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j11_dtests_repeat:
++ j8_utests_compression:
+ docker:
- - image: apache/cassandra-testing-ubuntu2004-java11:latest
++ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
++ - run:
++ name: Determine unit Tests to Run
++ command: |
++ # reminder: this code (along with all the steps) is independently executed on every circle container
++ # so the goal here is to get the circleci script to return the tests *this* container will run
++ # which we do via the `circleci` cli tool.
++
++ rm -fr ~/cassandra-dtest/upgrade_tests
++ echo "***java tests***"
++
++ # get all of our unit test filenames
++ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
++
++ # split up the unit tests into groups based on the number of containers we have
++ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
++ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
++ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
++ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
++ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
- name: Clone Cassandra dtest Repository (via git)
- command: |
- git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- - run:
- name: Configure virtualenv and python Dependencies
++ name: Run Unit Tests (testclasslist-compression)
+ command: |
- # note, this should be super quick as all dependencies should be pre-installed in the docker image
- # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
- # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
- # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
- source ~/env3.6/bin/activate
++ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
- pip3 uninstall -y cqlsh
- pip3 freeze
- - run:
- name: Run repeated Python dtest
- no_output_timeout: 15m
- command: |
- if [ "${REPEATED_DTESTS}" == "<nil>" ]; then
- echo "Repeated dtest name hasn't been defined, exiting without running any test"
- elif [ "${REPEATED_DTESTS_COUNT}" == "<nil>" ]; then
- echo "Repeated dtest count hasn't been defined, exiting without running any test"
- elif [ "${REPEATED_DTESTS_COUNT}" -le 0 ]; then
- echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
- else
-
- # Calculate the number of test iterations to be run by the current parallel runner.
- # Since we are running the same test multiple times there is no need to use `circleci tests split`.
- count=$((${REPEATED_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (${REPEATED_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
- count=$((count+1))
- fi
-
- if (($count <= 0)); then
- echo "No tests to run in this runner"
- else
- echo "Running ${REPEATED_DTESTS} $count times"
-
- source ~/env3.6/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
-
- java -version
- cd ~/cassandra-dtest
- mkdir -p /tmp/dtest
-
- echo "env: $(env)"
- echo "** done env"
- mkdir -p /tmp/results/dtests
-
- tests_arg=$(echo ${REPEATED_DTESTS} | sed -e "s/,/ /g")
-
- stop_on_failure_arg=""
- if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
- stop_on_failure_arg="-x"
- fi
-
- vnodes_args=""
- if false; then
- vnodes_args="--use-vnodes --num-tokens=16"
- fi
-
- upgrade_arg=""
- if false; then
- upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
- fi
-
- # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
- set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
- fi
++ time mv ~/cassandra /tmp
++ cd /tmp/cassandra
++ if [ -d ~/dtest_jars ]; then
++ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
++ fi
++ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
++ if [ -z "$test_timeout" ]; then
++ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
++ ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
++ no_output_timeout: 15m
+ - store_test_results:
- path: /tmp/results
++ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
- path: /tmp/dtest
- destination: dtest
++ path: /tmp/cassandra/build/test/output
++ destination: junitxml
+ - store_artifacts:
- path: ~/cassandra-dtest/logs
- destination: dtest_logs
++ path: /tmp/cassandra/build/test/logs
++ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - CASSANDRA_USE_JDK11: true
- j8_cqlsh_dtests_py38:
++ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ j11_utests_long:
+ docker:
- - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
++ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
- parallelism: 4
++ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
- name: Clone Cassandra dtest Repository (via git)
++ name: Run Unit Tests (long-test)
+ command: |
- git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- - run:
- name: Configure virtualenv and python Dependencies
- command: |
- # note, this should be super quick as all dependencies should be pre-installed in the docker image
- # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
- # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
- # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
- source ~/env3.8/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
- pip3 uninstall -y cqlsh
- pip3 freeze
- - run:
- name: Determine Tests to Run (j8_without_vnodes)
- no_output_timeout: 5m
- command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-i [...]
- - run:
- name: Run dtests (j8_without_vnodes)
- no_output_timeout: 15m
- command: |
- echo "cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt"
- cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt
-
- source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
- if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then
- export CQLSH_PYTHON=/usr/bin/python3.8
++ time mv ~/cassandra /tmp
++ cd /tmp/cassandra
++ if [ -d ~/dtest_jars ]; then
++ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
-
- java -version
- cd ~/cassandra-dtest
- mkdir -p /tmp/dtest
-
- echo "env: $(env)"
- echo "** done env"
- mkdir -p /tmp/results/dtests
- # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
- export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`
- set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
++ ant long-test
++ no_output_timeout: 15m
+ - store_test_results:
- path: /tmp/results
++ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
- path: /tmp/dtest
- destination: dtest_j8_without_vnodes
++ path: /tmp/cassandra/build/test/output
++ destination: junitxml
+ - store_artifacts:
- path: ~/cassandra-dtest/logs
- destination: dtest_j8_without_vnodes_logs
++ path: /tmp/cassandra/build/test/logs
++ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_dtests_vnode_repeat:
++ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - CASSANDRA_USE_JDK11: true
++ j8_unit_tests_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
- name: Clone Cassandra dtest Repository (via git)
- command: |
- git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- - run:
- name: Configure virtualenv and python Dependencies
++ name: Log Environment Information
+ command: |
- # note, this should be super quick as all dependencies should be pre-installed in the docker image
- # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
- # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
- # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
- source ~/env3.6/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
- pip3 uninstall -y cqlsh
- pip3 freeze
++ echo '*** id ***'
++ id
++ echo '*** cat /proc/cpuinfo ***'
++ cat /proc/cpuinfo
++ echo '*** free -m ***'
++ free -m
++ echo '*** df -m ***'
++ df -m
++ echo '*** ifconfig -a ***'
++ ifconfig -a
++ echo '*** uname -a ***'
++ uname -a
++ echo '*** mount ***'
++ mount
++ echo '*** env ***'
++ env
++ echo '*** java ***'
++ which java
++ java -version
+ - run:
- name: Run repeated Python dtest
++ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: |
- if [ "${REPEATED_DTESTS}" == "<nil>" ]; then
- echo "Repeated dtest name hasn't been defined, exiting without running any test"
- elif [ "${REPEATED_DTESTS_COUNT}" == "<nil>" ]; then
- echo "Repeated dtest count hasn't been defined, exiting without running any test"
- elif [ "${REPEATED_DTESTS_COUNT}" -le 0 ]; then
- echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
- else
-
- # Calculate the number of test iterations to be run by the current parallel runner.
- # Since we are running the same test multiple times there is no need to use `circleci tests split`.
- count=$((${REPEATED_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (${REPEATED_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
- count=$((count+1))
- fi
-
- if (($count <= 0)); then
- echo "No tests to run in this runner"
- else
- echo "Running ${REPEATED_DTESTS} $count times"
++ set -x
++ export PATH=$JAVA_HOME/bin:$PATH
++ time mv ~/cassandra /tmp
++ cd /tmp/cassandra
++ if [ -d ~/dtest_jars ]; then
++ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
++ fi
+
- source ~/env3.6/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
++ # Calculate the number of test iterations to be run by the current parallel runner.
++ count=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))
++ if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
++ count=$((count+1))
++ fi
+
- java -version
- cd ~/cassandra-dtest
- mkdir -p /tmp/dtest
++ # Put manually specified tests and automatically detected tests together, removing duplicates
++ tests=$(echo ${REPEATED_UTESTS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
++ echo "Tests to be repeated: ${tests}"
+
- echo "env: $(env)"
- echo "** done env"
- mkdir -p /tmp/results/dtests
++ # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
++ target=testsome
++ testtag=""
++ if [[ $target == "test-cdc" ]]; then
++ testtag="cdc"
++ elif [[ $target == "test-compression" ]]; then
++ testtag="compression"
++ elif [[ $target == "test-system-keyspace-directory" ]]; then
++ testtag="system_keyspace_directory"
++ fi
+
- tests_arg=$(echo ${REPEATED_DTESTS} | sed -e "s/,/ /g")
++ # Run each test class as many times as requested.
++ exit_code="$?"
++ for test in $tests; do
+
- stop_on_failure_arg=""
- if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
- stop_on_failure_arg="-x"
++ # Split class and method names from the test name
++ if [[ $test =~ "#" ]]; then
++ class=${test%"#"*}
++ method=${test#*"#"}
++ else
++ class=$test
++ method=""
+ fi
+
- vnodes_args=""
- if true; then
- vnodes_args="--use-vnodes --num-tokens=16"
++ # Prepare the -Dtest.name argument.
++ # It can be the fully qualified class name or the short class name, depending on the target.
++ if [[ $target == "test" || \
++ $target == "test-cdc" || \
++ $target == "test-compression" || \
++ $target == "test-system-keyspace-directory" || \
++ $target == "fqltool-test" || \
++ $target == "long-test" || \
++ $target == "stress-test" ]]; then
++ name_arg="-Dtest.name=${class##*.}"
++ else
++ name_arg="-Dtest.name=$class"
+ fi
+
- upgrade_arg=""
- if false; then
- upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
++ # Prepare the -Dtest.methods argument, which is optional
++ if [[ $method == "" ]]; then
++ methods_arg=""
++ else
++ methods_arg="-Dtest.methods=$method"
+ fi
+
- # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
- set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
- fi
- fi
++ for i in $(seq -w 1 $count); do
++ echo "Running test $test, iteration $i of $count"
++
++ # run the test
++ status="passes"
++ if !( set -o pipefail && \
++ ant testsome $name_arg $methods_arg -Dno-build-test=true | \
++ tee stdout.txt \
++ ); then
++ status="fails"
++ exit_code=1
++ fi
++
++ # move the stdout output file
++ dest=/tmp/results/repeated_utests/stdout/${status}/${i}
++ mkdir -p $dest
++ mv stdout.txt $dest/${test}.txt
++
++ # move the XML output files
++ source=build/test/output/${testtag}
++ dest=/tmp/results/repeated_utests/output/${status}/${i}
++ mkdir -p $dest
++ if [[ -d $source && -n "$(ls $source)" ]]; then
++ mv $source/* $dest/
++ fi
++
++ # move the log files
++ source=build/test/logs/${testtag}
++ dest=/tmp/results/repeated_utests/logs/${status}/${i}
++ mkdir -p $dest
++ if [[ -d $source && -n "$(ls $source)" ]]; then
++ mv $source/* $dest/
++ fi
++
++ # maybe stop iterations on test failure
++ if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
++ break
++ fi
++ done
++ done
++ (exit ${exit_code})
+ - store_test_results:
- path: /tmp/results
++ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
- path: /tmp/dtest
- destination: dtest
++ path: /tmp/results/repeated_utests/stdout
++ destination: stdout
+ - store_artifacts:
- path: ~/cassandra-dtest/logs
- destination: dtest_logs
++ path: /tmp/results/repeated_utests/output
++ destination: junitxml
++ - store_artifacts:
++ path: /tmp/results/repeated_utests/logs
++ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j11_cqlsh_dtests_py3_vnode:
++ j8_cqlsh-dtests-py2-no-vnodes:
+ docker:
- - image: apache/cassandra-testing-ubuntu2004-java11:latest
++ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
- name: Determine Tests to Run (j11_with_vnodes)
++ name: Determine Tests to Run (j8_without_vnodes)
+ no_output_timeout: 5m
- command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip [...]
++ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-i [...]
+ - run:
- name: Run dtests (j11_with_vnodes)
++ name: Run dtests (j8_without_vnodes)
+ no_output_timeout: 15m
+ command: |
- echo "cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt"
- cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt
++ echo "cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt"
++ cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
- if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then
- export CQLSH_PYTHON=/usr/bin/python3.6
++ if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then
++ export CQLSH_PYTHON=/usr/bin/python2.7
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
- export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt`
- set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
++ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`
++ set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
- destination: dtest_j11_with_vnodes
++ destination: dtest_j8_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
- destination: dtest_j11_with_vnodes_logs
++ destination: dtest_j8_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - CASSANDRA_USE_JDK11: true
- utests_fqltool_repeat:
++ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ j11_utests_stress:
+ docker:
- - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
++ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
- parallelism: 4
++ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
- name: Log Environment Information
- command: |
- echo '*** id ***'
- id
- echo '*** cat /proc/cpuinfo ***'
- cat /proc/cpuinfo
- echo '*** free -m ***'
- free -m
- echo '*** df -m ***'
- df -m
- echo '*** ifconfig -a ***'
- ifconfig -a
- echo '*** uname -a ***'
- uname -a
- echo '*** mount ***'
- mount
- echo '*** env ***'
- env
- echo '*** java ***'
- which java
- java -version
- - run:
- name: Repeatedly run new or modifed JUnit tests
- no_output_timeout: 15m
++ name: Run Unit Tests (stress-test)
+ command: |
- set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
-
- # Calculate the number of test iterations to be run by the current parallel runner.
- count=$((${REPEATED_UTESTS_FQLTOOL_COUNT} / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_FQLTOOL_COUNT} % CIRCLE_NODE_TOTAL))); then
- count=$((count+1))
- fi
-
- # Put manually specified tests and automatically detected tests together, removing duplicates
- tests=$(echo ${REPEATED_UTESTS_FQLTOOL} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
- echo "Tests to be repeated: ${tests}"
-
- # Run each test class as many times as requested.
- exit_code="$?"
- for test in $tests; do
-
- # Split class and method names from the test name
- if [[ $test =~ "#" ]]; then
- class=${test%"#"*}
- method=${test#*"#"}
- else
- class=$test
- method=""
- fi
-
- # Prepare the -Dtest.name argument.
- # It can be the fully qualified class name or the short class name, depending on the target.
- if [[ $target == "test" || \
- $target == "test-cdc" || \
- $target == "test-compression" || \
- $target == "test-system-keyspace-directory" || \
- $target == "fqltool-test" || \
- $target == "long-test" || \
- $target == "stress-test" ]]; then
- name_arg="-Dtest.name=${class##*.}"
- else
- name_arg="-Dtest.name=$class"
- fi
-
- # Prepare the -Dtest.methods argument, which is optional
- if [ $method == "" ]; then
- methods_arg=""
- else
- methods_arg="-Dtest.methods=$method"
- fi
-
- for i in $(seq -w 1 $count); do
- echo "Running test $test, iteration $i of $count"
-
- # run the test
- status="passes"
- if !( set -o pipefail && \
- ant fqltool-test $name_arg $methods_arg -Dno-build-test=true | \
- tee stdout.txt \
- ); then
- status="fails"
- exit_code=1
- fi
-
- # move the stdout output file
- dest=/tmp/results/repeated_utests/stdout/${status}/${i}
- mkdir -p $dest
- mv stdout.txt $dest/${test}.txt
-
- # move the XML output files
- source=build/test/output
- dest=/tmp/results/repeated_utests/output/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # move the log files
- source=build/test/logs
- dest=/tmp/results/repeated_utests/logs/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # maybe stop iterations on test failure
- if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
- break
- fi
- done
- done
- (exit ${exit_code})
++ ant stress-test
++ no_output_timeout: 15m
+ - store_test_results:
- path: /tmp/results/repeated_utests/output
- - store_artifacts:
- path: /tmp/results/repeated_utests/stdout
- destination: stdout
++ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
- path: /tmp/results/repeated_utests/output
++ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
- path: /tmp/results/repeated_utests/logs
++ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_upgrade_dtests:
++ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - CASSANDRA_USE_JDK11: true
++ j8_upgrade_dtests_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
- name: Determine Tests to Run (j8_upgradetests_without_vnodes)
- no_output_timeout: 5m
- command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_upgradetests_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --ex [...]
- - run:
- name: Run dtests (j8_upgradetests_without_vnodes)
++ name: Run repeated Python dtest
+ no_output_timeout: 15m
- command: "echo \"cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually [...]
++ command: |
++ if [ "${REPEATED_UPGRADE_DTESTS}" == "<nil>" ]; then
++ echo "Repeated dtest name hasn't been defined, exiting without running any test"
++ elif [ "${REPEATED_UPGRADE_DTESTS_COUNT}" == "<nil>" ]; then
++ echo "Repeated dtest count hasn't been defined, exiting without running any test"
++ elif [ "${REPEATED_UPGRADE_DTESTS_COUNT}" -le 0 ]; then
++ echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
++ else
++
++ # Calculate the number of test iterations to be run by the current parallel runner.
++ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
++ count=$((${REPEATED_UPGRADE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
++ if (($CIRCLE_NODE_INDEX < (${REPEATED_UPGRADE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
++ count=$((count+1))
++ fi
++
++ if (($count <= 0)); then
++ echo "No tests to run in this runner"
++ else
++ echo "Running ${REPEATED_UPGRADE_DTESTS} $count times"
++
++ source ~/env3.6/bin/activate
++ export PATH=$JAVA_HOME/bin:$PATH
++
++ java -version
++ cd ~/cassandra-dtest
++ mkdir -p /tmp/dtest
++
++ echo "env: $(env)"
++ echo "** done env"
++ mkdir -p /tmp/results/dtests
++
++ tests_arg=$(echo ${REPEATED_UPGRADE_DTESTS} | sed -e "s/,/ /g")
++
++ stop_on_failure_arg=""
++ if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
++ stop_on_failure_arg="-x"
++ fi
++
++ vnodes_args=""
++ if false; then
++ vnodes_args="--use-vnodes --num-tokens=16"
++ fi
++
++ upgrade_arg=""
++ if true; then
++ upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
++ fi
++
++ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
++ set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
++ fi
++ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
- destination: dtest_j8_upgradetests_without_vnodes
++ destination: dtest
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
- destination: dtest_j8_upgradetests_without_vnodes_logs
++ destination: dtest_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_jvm_upgrade_dtests_repeat:
++ j11_cqlsh-dtests-py2-with-vnodes:
+ docker:
- - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
++ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
- name: Log Environment Information
++ name: Clone Cassandra dtest Repository (via git)
+ command: |
- echo '*** id ***'
- id
- echo '*** cat /proc/cpuinfo ***'
- cat /proc/cpuinfo
- echo '*** free -m ***'
- free -m
- echo '*** df -m ***'
- df -m
- echo '*** ifconfig -a ***'
- ifconfig -a
- echo '*** uname -a ***'
- uname -a
- echo '*** mount ***'
- mount
- echo '*** env ***'
- env
- echo '*** java ***'
- which java
- java -version
++ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
- name: Repeatedly run new or modifed JUnit tests
++ name: Configure virtualenv and python Dependencies
++ command: |
++ # note, this should be super quick as all dependencies should be pre-installed in the docker image
++ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
++ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
++ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
++ source ~/env3.6/bin/activate
++ export PATH=$JAVA_HOME/bin:$PATH
++ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
++ pip3 uninstall -y cqlsh
++ pip3 freeze
++ - run:
++ name: Determine Tests to Run (j11_with_vnodes)
++ no_output_timeout: 5m
++ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip [...]
++ - run:
++ name: Run dtests (j11_with_vnodes)
+ no_output_timeout: 15m
+ command: |
- set -x
++ echo "cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt"
++ cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt
++
++ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- if [ -d ~/dtest_jars ]; then
++ if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then
++ export CQLSH_PYTHON=/usr/bin/python2.7
++ fi
++
++ java -version
++ cd ~/cassandra-dtest
++ mkdir -p /tmp/dtest
++
++ echo "env: $(env)"
++ echo "** done env"
++ mkdir -p /tmp/results/dtests
++ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
++ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt`
++ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
++ - store_test_results:
++ path: /tmp/results
++ - store_artifacts:
++ path: /tmp/dtest
++ destination: dtest_j11_with_vnodes
++ - store_artifacts:
++ path: ~/cassandra-dtest/logs
++ destination: dtest_j11_with_vnodes_logs
++ environment:
++ - ANT_HOME: /usr/share/ant
++ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ - LANG: en_US.UTF-8
++ - KEEP_TEST_DIR: true
++ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
++ - PYTHONIOENCODING: utf-8
++ - PYTHONUNBUFFERED: true
++ - CASS_DRIVER_NO_EXTENSIONS: true
++ - CASS_DRIVER_NO_CYTHON: true
++ - CASSANDRA_SKIP_SYNC: true
++ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
++ - DTEST_BRANCH: trunk
++ - CCM_MAX_HEAP_SIZE: 1024M
++ - CCM_HEAP_NEWSIZE: 256M
++ - REPEATED_TESTS_STOP_ON_FAILURE: false
++ - REPEATED_UTESTS: null
++ - REPEATED_UTESTS_COUNT: 500
++ - REPEATED_UTESTS_FQLTOOL: null
++ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
++ - REPEATED_UTESTS_LONG: null
++ - REPEATED_UTESTS_LONG_COUNT: 100
++ - REPEATED_UTESTS_STRESS: null
++ - REPEATED_UTESTS_STRESS_COUNT: 500
++ - REPEATED_JVM_DTESTS: null
++ - REPEATED_JVM_DTESTS_COUNT: 500
++ - REPEATED_JVM_UPGRADE_DTESTS: null
++ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
++ - REPEATED_DTESTS: null
++ - REPEATED_DTESTS_COUNT: 500
++ - REPEATED_UPGRADE_DTESTS: null
++ - REPEATED_UPGRADE_DTESTS_COUNT: 25
++ - REPEATED_ANT_TEST_TARGET: testsome
++ - REPEATED_ANT_TEST_CLASS: null
++ - REPEATED_ANT_TEST_METHODS: null
++ - REPEATED_ANT_TEST_COUNT: 500
++ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - CASSANDRA_USE_JDK11: true
++ j11_utests_cdc_repeat:
++ docker:
++ - image: apache/cassandra-testing-ubuntu2004-java11:latest
++ resource_class: medium
++ working_directory: ~/
++ shell: /bin/bash -eo pipefail -l
++ parallelism: 4
++ steps:
++ - attach_workspace:
++ at: /home/cassandra
++ - run:
++ name: Log Environment Information
++ command: |
++ echo '*** id ***'
++ id
++ echo '*** cat /proc/cpuinfo ***'
++ cat /proc/cpuinfo
++ echo '*** free -m ***'
++ free -m
++ echo '*** df -m ***'
++ df -m
++ echo '*** ifconfig -a ***'
++ ifconfig -a
++ echo '*** uname -a ***'
++ uname -a
++ echo '*** mount ***'
++ mount
++ echo '*** env ***'
++ env
++ echo '*** java ***'
++ which java
++ java -version
++ - run:
++ name: Repeatedly run new or modifed JUnit tests
++ no_output_timeout: 15m
++ command: |
++ set -x
++ export PATH=$JAVA_HOME/bin:$PATH
++ time mv ~/cassandra /tmp
++ cd /tmp/cassandra
++ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
- count=$((${REPEATED_JVM_UPGRADE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_UPGRADE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
++ count=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))
++ if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ # Put manually specified tests and automatically detected tests together, removing duplicates
- tests=$(echo ${REPEATED_JVM_UPGRADE_DTESTS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
++ tests=$(echo ${REPEATED_UTESTS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
+ echo "Tests to be repeated: ${tests}"
+
++ # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
++ target=test-cdc
++ testtag=""
++ if [[ $target == "test-cdc" ]]; then
++ testtag="cdc"
++ elif [[ $target == "test-compression" ]]; then
++ testtag="compression"
++ elif [[ $target == "test-system-keyspace-directory" ]]; then
++ testtag="system_keyspace_directory"
++ fi
++
+ # Run each test class as many times as requested.
+ exit_code="$?"
+ for test in $tests; do
+
+ # Split class and method names from the test name
+ if [[ $test =~ "#" ]]; then
+ class=${test%"#"*}
+ method=${test#*"#"}
+ else
+ class=$test
+ method=""
+ fi
+
+ # Prepare the -Dtest.name argument.
+ # It can be the fully qualified class name or the short class name, depending on the target.
+ if [[ $target == "test" || \
+ $target == "test-cdc" || \
+ $target == "test-compression" || \
+ $target == "test-system-keyspace-directory" || \
+ $target == "fqltool-test" || \
+ $target == "long-test" || \
+ $target == "stress-test" ]]; then
+ name_arg="-Dtest.name=${class##*.}"
+ else
+ name_arg="-Dtest.name=$class"
+ fi
+
+ # Prepare the -Dtest.methods argument, which is optional
- if [ $method == "" ]; then
++ if [[ $method == "" ]]; then
+ methods_arg=""
+ else
+ methods_arg="-Dtest.methods=$method"
+ fi
+
+ for i in $(seq -w 1 $count); do
+ echo "Running test $test, iteration $i of $count"
+
+ # run the test
+ status="passes"
+ if !( set -o pipefail && \
- ant test-jvm-dtest-some $name_arg $methods_arg -Dno-build-test=true | \
++ ant test-cdc $name_arg $methods_arg -Dno-build-test=true | \
+ tee stdout.txt \
+ ); then
+ status="fails"
+ exit_code=1
+ fi
+
+ # move the stdout output file
+ dest=/tmp/results/repeated_utests/stdout/${status}/${i}
+ mkdir -p $dest
+ mv stdout.txt $dest/${test}.txt
+
+ # move the XML output files
- source=build/test/output
++ source=build/test/output/${testtag}
+ dest=/tmp/results/repeated_utests/output/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # move the log files
- source=build/test/logs
++ source=build/test/logs/${testtag}
+ dest=/tmp/results/repeated_utests/logs/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # maybe stop iterations on test failure
+ if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
+ break
+ fi
+ done
+ done
+ (exit ${exit_code})
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j11_cqlsh_dtests_py38_vnode:
++ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - CASSANDRA_USE_JDK11: true
++ j11_dtests_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
++ - run:
++ name: Log Environment Information
++ command: |
++ echo '*** id ***'
++ id
++ echo '*** cat /proc/cpuinfo ***'
++ cat /proc/cpuinfo
++ echo '*** free -m ***'
++ free -m
++ echo '*** df -m ***'
++ df -m
++ echo '*** ifconfig -a ***'
++ ifconfig -a
++ echo '*** uname -a ***'
++ uname -a
++ echo '*** mount ***'
++ mount
++ echo '*** env ***'
++ env
++ echo '*** java ***'
++ which java
++ java -version
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
- source ~/env3.8/bin/activate
++ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
- name: Determine Tests to Run (j11_with_vnodes)
- no_output_timeout: 5m
- command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip [...]
- - run:
- name: Run dtests (j11_with_vnodes)
++ name: Run repeated Python dtest
+ no_output_timeout: 15m
+ command: |
- echo "cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt"
- cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt
++ if [ "${REPEATED_DTESTS}" == "<nil>" ]; then
++ echo "Repeated dtest name hasn't been defined, exiting without running any test"
++ elif [ "${REPEATED_DTESTS_COUNT}" == "<nil>" ]; then
++ echo "Repeated dtest count hasn't been defined, exiting without running any test"
++ elif [ "${REPEATED_DTESTS_COUNT}" -le 0 ]; then
++ echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
++ else
+
- source ~/env3.8/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
- if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then
- export CQLSH_PYTHON=/usr/bin/python3.8
- fi
++ # Calculate the number of test iterations to be run by the current parallel runner.
++ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
++ count=$((${REPEATED_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
++ if (($CIRCLE_NODE_INDEX < (${REPEATED_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
++ count=$((count+1))
++ fi
+
- java -version
- cd ~/cassandra-dtest
- mkdir -p /tmp/dtest
++ if (($count <= 0)); then
++ echo "No tests to run in this runner"
++ else
++ echo "Running ${REPEATED_DTESTS} $count times"
+
- echo "env: $(env)"
- echo "** done env"
- mkdir -p /tmp/results/dtests
- # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
- export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt`
- set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
++ source ~/env3.6/bin/activate
++ export PATH=$JAVA_HOME/bin:$PATH
++
++ java -version
++ cd ~/cassandra-dtest
++ mkdir -p /tmp/dtest
++
++ echo "env: $(env)"
++ echo "** done env"
++ mkdir -p /tmp/results/dtests
++
++ tests_arg=$(echo ${REPEATED_DTESTS} | sed -e "s/,/ /g")
++
++ stop_on_failure_arg=""
++ if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
++ stop_on_failure_arg="-x"
++ fi
++
++ vnodes_args=""
++ if false; then
++ vnodes_args="--use-vnodes --num-tokens=16"
++ fi
++
++ upgrade_arg=""
++ if false; then
++ upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
++ fi
++
++ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
++ set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
++ fi
++ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
- destination: dtest_j11_with_vnodes
++ destination: dtest
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
- destination: dtest_j11_with_vnodes_logs
++ destination: dtest_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
- j11_jvm_dtests_repeat:
++ j8_utests_fqltool_repeat:
+ docker:
- - image: apache/cassandra-testing-ubuntu2004-java11:latest
++ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
- count=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
++ count=$((${REPEATED_UTESTS_FQLTOOL_COUNT} / CIRCLE_NODE_TOTAL))
++ if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_FQLTOOL_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ # Put manually specified tests and automatically detected tests together, removing duplicates
- tests=$(echo ${REPEATED_JVM_DTESTS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
++ tests=$(echo ${REPEATED_UTESTS_FQLTOOL} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
+ echo "Tests to be repeated: ${tests}"
+
++ # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
++ target=fqltool-test
++ testtag=""
++ if [[ $target == "test-cdc" ]]; then
++ testtag="cdc"
++ elif [[ $target == "test-compression" ]]; then
++ testtag="compression"
++ elif [[ $target == "test-system-keyspace-directory" ]]; then
++ testtag="system_keyspace_directory"
++ fi
++
+ # Run each test class as many times as requested.
+ exit_code="$?"
+ for test in $tests; do
+
+ # Split class and method names from the test name
+ if [[ $test =~ "#" ]]; then
+ class=${test%"#"*}
+ method=${test#*"#"}
+ else
+ class=$test
+ method=""
+ fi
+
+ # Prepare the -Dtest.name argument.
+ # It can be the fully qualified class name or the short class name, depending on the target.
+ if [[ $target == "test" || \
+ $target == "test-cdc" || \
+ $target == "test-compression" || \
+ $target == "test-system-keyspace-directory" || \
+ $target == "fqltool-test" || \
+ $target == "long-test" || \
+ $target == "stress-test" ]]; then
+ name_arg="-Dtest.name=${class##*.}"
+ else
+ name_arg="-Dtest.name=$class"
+ fi
+
+ # Prepare the -Dtest.methods argument, which is optional
- if [ $method == "" ]; then
++ if [[ $method == "" ]]; then
+ methods_arg=""
+ else
+ methods_arg="-Dtest.methods=$method"
+ fi
+
+ for i in $(seq -w 1 $count); do
+ echo "Running test $test, iteration $i of $count"
+
+ # run the test
+ status="passes"
+ if !( set -o pipefail && \
- ant test-jvm-dtest-some $name_arg $methods_arg -Dno-build-test=true | \
++ ant fqltool-test $name_arg $methods_arg -Dno-build-test=true | \
+ tee stdout.txt \
+ ); then
+ status="fails"
+ exit_code=1
+ fi
+
+ # move the stdout output file
+ dest=/tmp/results/repeated_utests/stdout/${status}/${i}
+ mkdir -p $dest
+ mv stdout.txt $dest/${test}.txt
+
+ # move the XML output files
- source=build/test/output
++ source=build/test/output/${testtag}
+ dest=/tmp/results/repeated_utests/output/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # move the log files
- source=build/test/logs
++ source=build/test/logs/${testtag}
+ dest=/tmp/results/repeated_utests/logs/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # maybe stop iterations on test failure
+ if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
+ break
+ fi
+ done
+ done
+ (exit ${exit_code})
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - CASSANDRA_USE_JDK11: true
- utests_stress:
++ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ j11_utests_compression:
+ docker:
- - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
++ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
- parallelism: 1
++ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
- name: Run Unit Tests (stress-test)
- command: |
- export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- if [ -d ~/dtest_jars ]; then
- cp ~/dtest_jars/dtest* /tmp/cassandra/build/
- fi
- ant stress-test
- no_output_timeout: 15m
- - store_test_results:
- path: /tmp/cassandra/build/test/output/
- - store_artifacts:
- path: /tmp/cassandra/build/test/output
- destination: junitxml
- - store_artifacts:
- path: /tmp/cassandra/build/test/logs
- destination: logs
- environment:
- - ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - LANG: en_US.UTF-8
- - KEEP_TEST_DIR: true
- - DEFAULT_DIR: /home/cassandra/cassandra-dtest
- - PYTHONIOENCODING: utf-8
- - PYTHONUNBUFFERED: true
- - CASS_DRIVER_NO_EXTENSIONS: true
- - CASS_DRIVER_NO_CYTHON: true
- - CASSANDRA_SKIP_SYNC: true
- - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: trunk
- - CCM_MAX_HEAP_SIZE: 1024M
- - CCM_HEAP_NEWSIZE: 256M
- - REPEATED_TESTS_STOP_ON_FAILURE: false
- - REPEATED_UTESTS: null
- - REPEATED_UTESTS_COUNT: 500
- - REPEATED_UTESTS_FQLTOOL: null
- - REPEATED_UTESTS_FQLTOOL_COUNT: 500
- - REPEATED_UTESTS_LONG: null
- - REPEATED_UTESTS_LONG_COUNT: 100
- - REPEATED_UTESTS_STRESS: null
- - REPEATED_UTESTS_STRESS_COUNT: 500
- - REPEATED_JVM_DTESTS: null
- - REPEATED_JVM_DTESTS_COUNT: 500
- - REPEATED_JVM_UPGRADE_DTESTS: null
- - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- - REPEATED_DTESTS: null
- - REPEATED_DTESTS_COUNT: 500
- - REPEATED_UPGRADE_DTESTS: null
- - REPEATED_UPGRADE_DTESTS_COUNT: 25
- - REPEATED_ANT_TEST_TARGET: testsome
- - REPEATED_ANT_TEST_CLASS: null
- - REPEATED_ANT_TEST_METHODS: null
- - REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_unit_tests:
- docker:
- - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
- resource_class: medium
- working_directory: ~/
- shell: /bin/bash -eo pipefail -l
- parallelism: 4
- steps:
- - attach_workspace:
- at: /home/cassandra
- - run:
- name: Determine unit Tests to Run
++ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
- name: Run Unit Tests (testclasslist)
++ name: Run Unit Tests (testclasslist-compression)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
- ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
++ ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j11_jvm_dtests:
++ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - CASSANDRA_USE_JDK11: true
++ j8_cqlsh_dtests_py38:
+ docker:
- - image: apache/cassandra-testing-ubuntu2004-java11:latest
++ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
- parallelism: 1
++ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
- name: Determine distributed Tests to Run
++ name: Clone Cassandra dtest Repository (via git)
+ command: |
- # reminder: this code (along with all the steps) is independently executed on every circle container
- # so the goal here is to get the circleci script to return the tests *this* container will run
- # which we do via the `circleci` cli tool.
-
- rm -fr ~/cassandra-dtest/upgrade_tests
- echo "***java tests***"
-
- # get all of our unit test filenames
- set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt
-
- # split up the unit tests into groups based on the number of containers we have
- set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
- set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep -v upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
- echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
- cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
- no_output_timeout: 15m
++ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
- name: Log Environment Information
++ name: Configure virtualenv and python Dependencies
+ command: |
- echo '*** id ***'
- id
- echo '*** cat /proc/cpuinfo ***'
- cat /proc/cpuinfo
- echo '*** free -m ***'
- free -m
- echo '*** df -m ***'
- df -m
- echo '*** ifconfig -a ***'
- ifconfig -a
- echo '*** uname -a ***'
- uname -a
- echo '*** mount ***'
- mount
- echo '*** env ***'
- env
- echo '*** java ***'
- which java
- java -version
++ # note, this should be super quick as all dependencies should be pre-installed in the docker image
++ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
++ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
++ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
++ source ~/env3.8/bin/activate
++ export PATH=$JAVA_HOME/bin:$PATH
++ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
++ pip3 uninstall -y cqlsh
++ pip3 freeze
+ - run:
- name: Run Unit Tests (testclasslist)
++ name: Determine Tests to Run (j8_without_vnodes)
++ no_output_timeout: 5m
++ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-i [...]
++ - run:
++ name: Run dtests (j8_without_vnodes)
++ no_output_timeout: 15m
+ command: |
- set -x
++ echo "cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt"
++ cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt
++
++ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- if [ -d ~/dtest_jars ]; then
- cp ~/dtest_jars/dtest* /tmp/cassandra/build/
- fi
- test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true)
- if [ -z "$test_timeout" ]; then
- test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
++ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then
++ export CQLSH_PYTHON=/usr/bin/python3.8
+ fi
- ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed
- no_output_timeout: 15m
++
++ java -version
++ cd ~/cassandra-dtest
++ mkdir -p /tmp/dtest
++
++ echo "env: $(env)"
++ echo "** done env"
++ mkdir -p /tmp/results/dtests
++ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
++ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`
++ set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
- path: /tmp/cassandra/build/test/output/
++ path: /tmp/results
+ - store_artifacts:
- path: /tmp/cassandra/build/test/output
- destination: junitxml
++ path: /tmp/dtest
++ destination: dtest_j8_without_vnodes
+ - store_artifacts:
- path: /tmp/cassandra/build/test/logs
- destination: logs
++ path: ~/cassandra-dtest/logs
++ destination: dtest_j8_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - CASSANDRA_USE_JDK11: true
- j11_build:
++ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ j8_utests_system_keyspace_directory_repeat:
+ docker:
- - image: apache/cassandra-testing-ubuntu2004-java11:latest
++ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
- parallelism: 1
++ parallelism: 4
+ steps:
++ - attach_workspace:
++ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
- name: Clone Cassandra Repository (via git)
- command: |
- git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH https://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra
- - run:
- name: Build Cassandra
- command: |
- export PATH=$JAVA_HOME/bin:$PATH
- cd ~/cassandra
- # Loop to prevent failure due to maven-ant-tasks not downloading a jar..
- for x in $(seq 1 3); do
- ${ANT_HOME}/bin/ant clean realclean jar
- RETURN="$?"
- if [ "${RETURN}" -eq "0" ]; then
- break
- fi
- done
- # Exit, if we didn't build successfully
- if [ "${RETURN}" -ne "0" ]; then
- echo "Build failed with exit code: ${RETURN}"
- exit ${RETURN}
- fi
++ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
- - run:
- name: Run eclipse-warnings
+ command: |
++ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
- cd ~/cassandra
- ant eclipse-warnings
- - persist_to_workspace:
- root: /home/cassandra
- paths:
- - cassandra
- - .m2
- environment:
- - ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - LANG: en_US.UTF-8
- - KEEP_TEST_DIR: true
- - DEFAULT_DIR: /home/cassandra/cassandra-dtest
- - PYTHONIOENCODING: utf-8
- - PYTHONUNBUFFERED: true
- - CASS_DRIVER_NO_EXTENSIONS: true
- - CASS_DRIVER_NO_CYTHON: true
- - CASSANDRA_SKIP_SYNC: true
- - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: trunk
- - CCM_MAX_HEAP_SIZE: 1024M
- - CCM_HEAP_NEWSIZE: 256M
- - REPEATED_TESTS_STOP_ON_FAILURE: false
- - REPEATED_UTESTS: null
- - REPEATED_UTESTS_COUNT: 500
- - REPEATED_UTESTS_FQLTOOL: null
- - REPEATED_UTESTS_FQLTOOL_COUNT: 500
- - REPEATED_UTESTS_LONG: null
- - REPEATED_UTESTS_LONG_COUNT: 100
- - REPEATED_UTESTS_STRESS: null
- - REPEATED_UTESTS_STRESS_COUNT: 500
- - REPEATED_JVM_DTESTS: null
- - REPEATED_JVM_DTESTS_COUNT: 500
- - REPEATED_JVM_UPGRADE_DTESTS: null
- - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- - REPEATED_DTESTS: null
- - REPEATED_DTESTS_COUNT: 500
- - REPEATED_UPGRADE_DTESTS: null
- - REPEATED_UPGRADE_DTESTS_COUNT: 25
- - REPEATED_ANT_TEST_TARGET: testsome
- - REPEATED_ANT_TEST_CLASS: null
- - REPEATED_ANT_TEST_METHODS: null
- - REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - CASSANDRA_USE_JDK11: true
- j8_dtests:
- docker:
- - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
- resource_class: medium
- working_directory: ~/
- shell: /bin/bash -eo pipefail -l
++ time mv ~/cassandra /tmp
++ cd /tmp/cassandra
++ if [ -d ~/dtest_jars ]; then
++ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
++ fi
++
++ # Calculate the number of test iterations to be run by the current parallel runner.
++ count=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))
++ if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
++ count=$((count+1))
++ fi
++
++ # Put manually specified tests and automatically detected tests together, removing duplicates
++ tests=$(echo ${REPEATED_UTESTS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
++ echo "Tests to be repeated: ${tests}"
++
++ # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
++ target=test-system-keyspace-directory
++ testtag=""
++ if [[ $target == "test-cdc" ]]; then
++ testtag="cdc"
++ elif [[ $target == "test-compression" ]]; then
++ testtag="compression"
++ elif [[ $target == "test-system-keyspace-directory" ]]; then
++ testtag="system_keyspace_directory"
++ fi
++
++ # Run each test class as many times as requested.
++ exit_code="$?"
++ for test in $tests; do
++
++ # Split class and method names from the test name
++ if [[ $test =~ "#" ]]; then
++ class=${test%"#"*}
++ method=${test#*"#"}
++ else
++ class=$test
++ method=""
++ fi
++
++ # Prepare the -Dtest.name argument.
++ # It can be the fully qualified class name or the short class name, depending on the target.
++ if [[ $target == "test" || \
++ $target == "test-cdc" || \
++ $target == "test-compression" || \
++ $target == "test-system-keyspace-directory" || \
++ $target == "fqltool-test" || \
++ $target == "long-test" || \
++ $target == "stress-test" ]]; then
++ name_arg="-Dtest.name=${class##*.}"
++ else
++ name_arg="-Dtest.name=$class"
++ fi
++
++ # Prepare the -Dtest.methods argument, which is optional
++ if [[ $method == "" ]]; then
++ methods_arg=""
++ else
++ methods_arg="-Dtest.methods=$method"
++ fi
++
++ for i in $(seq -w 1 $count); do
++ echo "Running test $test, iteration $i of $count"
++
++ # run the test
++ status="passes"
++ if !( set -o pipefail && \
++ ant test-system-keyspace-directory $name_arg $methods_arg -Dno-build-test=true | \
++ tee stdout.txt \
++ ); then
++ status="fails"
++ exit_code=1
++ fi
++
++ # move the stdout output file
++ dest=/tmp/results/repeated_utests/stdout/${status}/${i}
++ mkdir -p $dest
++ mv stdout.txt $dest/${test}.txt
++
++ # move the XML output files
++ source=build/test/output/${testtag}
++ dest=/tmp/results/repeated_utests/output/${status}/${i}
++ mkdir -p $dest
++ if [[ -d $source && -n "$(ls $source)" ]]; then
++ mv $source/* $dest/
++ fi
++
++ # move the log files
++ source=build/test/logs/${testtag}
++ dest=/tmp/results/repeated_utests/logs/${status}/${i}
++ mkdir -p $dest
++ if [[ -d $source && -n "$(ls $source)" ]]; then
++ mv $source/* $dest/
++ fi
++
++ # maybe stop iterations on test failure
++ if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
++ break
++ fi
++ done
++ done
++ (exit ${exit_code})
++ - store_test_results:
++ path: /tmp/results/repeated_utests/output
++ - store_artifacts:
++ path: /tmp/results/repeated_utests/stdout
++ destination: stdout
++ - store_artifacts:
++ path: /tmp/results/repeated_utests/output
++ destination: junitxml
++ - store_artifacts:
++ path: /tmp/results/repeated_utests/logs
++ destination: logs
++ environment:
++ - ANT_HOME: /usr/share/ant
++ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ - LANG: en_US.UTF-8
++ - KEEP_TEST_DIR: true
++ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
++ - PYTHONIOENCODING: utf-8
++ - PYTHONUNBUFFERED: true
++ - CASS_DRIVER_NO_EXTENSIONS: true
++ - CASS_DRIVER_NO_CYTHON: true
++ - CASSANDRA_SKIP_SYNC: true
++ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
++ - DTEST_BRANCH: trunk
++ - CCM_MAX_HEAP_SIZE: 1024M
++ - CCM_HEAP_NEWSIZE: 256M
++ - REPEATED_TESTS_STOP_ON_FAILURE: false
++ - REPEATED_UTESTS: null
++ - REPEATED_UTESTS_COUNT: 500
++ - REPEATED_UTESTS_FQLTOOL: null
++ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
++ - REPEATED_UTESTS_LONG: null
++ - REPEATED_UTESTS_LONG_COUNT: 100
++ - REPEATED_UTESTS_STRESS: null
++ - REPEATED_UTESTS_STRESS_COUNT: 500
++ - REPEATED_JVM_DTESTS: null
++ - REPEATED_JVM_DTESTS_COUNT: 500
++ - REPEATED_JVM_UPGRADE_DTESTS: null
++ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
++ - REPEATED_DTESTS: null
++ - REPEATED_DTESTS_COUNT: 500
++ - REPEATED_UPGRADE_DTESTS: null
++ - REPEATED_UPGRADE_DTESTS_COUNT: 25
++ - REPEATED_ANT_TEST_TARGET: testsome
++ - REPEATED_ANT_TEST_CLASS: null
++ - REPEATED_ANT_TEST_METHODS: null
++ - REPEATED_ANT_TEST_COUNT: 500
++ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ j8_utests_stress_repeat:
++ docker:
++ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
++ resource_class: medium
++ working_directory: ~/
++ shell: /bin/bash -eo pipefail -l
++ parallelism: 4
++ steps:
++ - attach_workspace:
++ at: /home/cassandra
++ - run:
++ name: Log Environment Information
++ command: |
++ echo '*** id ***'
++ id
++ echo '*** cat /proc/cpuinfo ***'
++ cat /proc/cpuinfo
++ echo '*** free -m ***'
++ free -m
++ echo '*** df -m ***'
++ df -m
++ echo '*** ifconfig -a ***'
++ ifconfig -a
++ echo '*** uname -a ***'
++ uname -a
++ echo '*** mount ***'
++ mount
++ echo '*** env ***'
++ env
++ echo '*** java ***'
++ which java
++ java -version
++ - run:
++ name: Repeatedly run new or modifed JUnit tests
++ no_output_timeout: 15m
++ command: |
++ set -x
++ export PATH=$JAVA_HOME/bin:$PATH
++ time mv ~/cassandra /tmp
++ cd /tmp/cassandra
++ if [ -d ~/dtest_jars ]; then
++ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
++ fi
++
++ # Calculate the number of test iterations to be run by the current parallel runner.
++ count=$((${REPEATED_UTESTS_STRESS_COUNT} / CIRCLE_NODE_TOTAL))
++ if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_STRESS_COUNT} % CIRCLE_NODE_TOTAL))); then
++ count=$((count+1))
++ fi
++
++ # Put manually specified tests and automatically detected tests together, removing duplicates
++ tests=$(echo ${REPEATED_UTESTS_STRESS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
++ echo "Tests to be repeated: ${tests}"
++
++ # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
++ target=stress-test-some
++ testtag=""
++ if [[ $target == "test-cdc" ]]; then
++ testtag="cdc"
++ elif [[ $target == "test-compression" ]]; then
++ testtag="compression"
++ elif [[ $target == "test-system-keyspace-directory" ]]; then
++ testtag="system_keyspace_directory"
++ fi
++
++ # Run each test class as many times as requested.
++ exit_code="$?"
++ for test in $tests; do
++
++ # Split class and method names from the test name
++ if [[ $test =~ "#" ]]; then
++ class=${test%"#"*}
++ method=${test#*"#"}
++ else
++ class=$test
++ method=""
++ fi
++
++ # Prepare the -Dtest.name argument.
++ # It can be the fully qualified class name or the short class name, depending on the target.
++ if [[ $target == "test" || \
++ $target == "test-cdc" || \
++ $target == "test-compression" || \
++ $target == "test-system-keyspace-directory" || \
++ $target == "fqltool-test" || \
++ $target == "long-test" || \
++ $target == "stress-test" ]]; then
++ name_arg="-Dtest.name=${class##*.}"
++ else
++ name_arg="-Dtest.name=$class"
++ fi
++
++ # Prepare the -Dtest.methods argument, which is optional
++ if [[ $method == "" ]]; then
++ methods_arg=""
++ else
++ methods_arg="-Dtest.methods=$method"
++ fi
++
++ for i in $(seq -w 1 $count); do
++ echo "Running test $test, iteration $i of $count"
++
++ # run the test
++ status="passes"
++ if !( set -o pipefail && \
++ ant stress-test-some $name_arg $methods_arg -Dno-build-test=true | \
++ tee stdout.txt \
++ ); then
++ status="fails"
++ exit_code=1
++ fi
++
++ # move the stdout output file
++ dest=/tmp/results/repeated_utests/stdout/${status}/${i}
++ mkdir -p $dest
++ mv stdout.txt $dest/${test}.txt
++
++ # move the XML output files
++ source=build/test/output/${testtag}
++ dest=/tmp/results/repeated_utests/output/${status}/${i}
++ mkdir -p $dest
++ if [[ -d $source && -n "$(ls $source)" ]]; then
++ mv $source/* $dest/
++ fi
++
++ # move the log files
++ source=build/test/logs/${testtag}
++ dest=/tmp/results/repeated_utests/logs/${status}/${i}
++ mkdir -p $dest
++ if [[ -d $source && -n "$(ls $source)" ]]; then
++ mv $source/* $dest/
++ fi
++
++ # maybe stop iterations on test failure
++ if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
++ break
++ fi
++ done
++ done
++ (exit ${exit_code})
++ - store_test_results:
++ path: /tmp/results/repeated_utests/output
++ - store_artifacts:
++ path: /tmp/results/repeated_utests/stdout
++ destination: stdout
++ - store_artifacts:
++ path: /tmp/results/repeated_utests/output
++ destination: junitxml
++ - store_artifacts:
++ path: /tmp/results/repeated_utests/logs
++ destination: logs
++ environment:
++ - ANT_HOME: /usr/share/ant
++ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ - LANG: en_US.UTF-8
++ - KEEP_TEST_DIR: true
++ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
++ - PYTHONIOENCODING: utf-8
++ - PYTHONUNBUFFERED: true
++ - CASS_DRIVER_NO_EXTENSIONS: true
++ - CASS_DRIVER_NO_CYTHON: true
++ - CASSANDRA_SKIP_SYNC: true
++ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
++ - DTEST_BRANCH: trunk
++ - CCM_MAX_HEAP_SIZE: 1024M
++ - CCM_HEAP_NEWSIZE: 256M
++ - REPEATED_TESTS_STOP_ON_FAILURE: false
++ - REPEATED_UTESTS: null
++ - REPEATED_UTESTS_COUNT: 500
++ - REPEATED_UTESTS_FQLTOOL: null
++ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
++ - REPEATED_UTESTS_LONG: null
++ - REPEATED_UTESTS_LONG_COUNT: 100
++ - REPEATED_UTESTS_STRESS: null
++ - REPEATED_UTESTS_STRESS_COUNT: 500
++ - REPEATED_JVM_DTESTS: null
++ - REPEATED_JVM_DTESTS_COUNT: 500
++ - REPEATED_JVM_UPGRADE_DTESTS: null
++ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
++ - REPEATED_DTESTS: null
++ - REPEATED_DTESTS_COUNT: 500
++ - REPEATED_UPGRADE_DTESTS: null
++ - REPEATED_UPGRADE_DTESTS_COUNT: 25
++ - REPEATED_ANT_TEST_TARGET: testsome
++ - REPEATED_ANT_TEST_CLASS: null
++ - REPEATED_ANT_TEST_METHODS: null
++ - REPEATED_ANT_TEST_COUNT: 500
++ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ j8_dtests_vnode_repeat:
++ docker:
++ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
++ resource_class: medium
++ working_directory: ~/
++ shell: /bin/bash -eo pipefail -l
++ parallelism: 4
++ steps:
++ - attach_workspace:
++ at: /home/cassandra
++ - run:
++ name: Clone Cassandra dtest Repository (via git)
++ command: |
++ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
++ - run:
++ name: Configure virtualenv and python Dependencies
++ command: |
++ # note, this should be super quick as all dependencies should be pre-installed in the docker image
++ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
++ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
++ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
++ source ~/env3.6/bin/activate
++ export PATH=$JAVA_HOME/bin:$PATH
++ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
++ pip3 uninstall -y cqlsh
++ pip3 freeze
++ - run:
++ name: Run repeated Python dtest
++ no_output_timeout: 15m
++ command: |
++ if [ "${REPEATED_DTESTS}" == "<nil>" ]; then
++ echo "Repeated dtest name hasn't been defined, exiting without running any test"
++ elif [ "${REPEATED_DTESTS_COUNT}" == "<nil>" ]; then
++ echo "Repeated dtest count hasn't been defined, exiting without running any test"
++ elif [ "${REPEATED_DTESTS_COUNT}" -le 0 ]; then
++ echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
+ else
+
- # Calculate the number of test iterations to be run by the current parallel runner.
- # Since we are running the same test multiple times there is no need to use `circleci tests split`.
- count=$((${REPEATED_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (${REPEATED_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
- count=$((count+1))
- fi
++ # Calculate the number of test iterations to be run by the current parallel runner.
++ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
++ count=$((${REPEATED_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
++ if (($CIRCLE_NODE_INDEX < (${REPEATED_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
++ count=$((count+1))
++ fi
++
++ if (($count <= 0)); then
++ echo "No tests to run in this runner"
++ else
++ echo "Running ${REPEATED_DTESTS} $count times"
++
++ source ~/env3.6/bin/activate
++ export PATH=$JAVA_HOME/bin:$PATH
++
++ java -version
++ cd ~/cassandra-dtest
++ mkdir -p /tmp/dtest
++
++ echo "env: $(env)"
++ echo "** done env"
++ mkdir -p /tmp/results/dtests
++
++ tests_arg=$(echo ${REPEATED_DTESTS} | sed -e "s/,/ /g")
++
++ stop_on_failure_arg=""
++ if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
++ stop_on_failure_arg="-x"
++ fi
++
++ vnodes_args=""
++ if true; then
++ vnodes_args="--use-vnodes --num-tokens=16"
++ fi
++
++ upgrade_arg=""
++ if false; then
++ upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
++ fi
++
++ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
++ set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
++ fi
++ fi
++ - store_test_results:
++ path: /tmp/results
++ - store_artifacts:
++ path: /tmp/dtest
++ destination: dtest
++ - store_artifacts:
++ path: ~/cassandra-dtest/logs
++ destination: dtest_logs
++ environment:
++ - ANT_HOME: /usr/share/ant
++ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ - LANG: en_US.UTF-8
++ - KEEP_TEST_DIR: true
++ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
++ - PYTHONIOENCODING: utf-8
++ - PYTHONUNBUFFERED: true
++ - CASS_DRIVER_NO_EXTENSIONS: true
++ - CASS_DRIVER_NO_CYTHON: true
++ - CASSANDRA_SKIP_SYNC: true
++ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
++ - DTEST_BRANCH: trunk
++ - CCM_MAX_HEAP_SIZE: 1024M
++ - CCM_HEAP_NEWSIZE: 256M
++ - REPEATED_TESTS_STOP_ON_FAILURE: false
++ - REPEATED_UTESTS: null
++ - REPEATED_UTESTS_COUNT: 500
++ - REPEATED_UTESTS_FQLTOOL: null
++ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
++ - REPEATED_UTESTS_LONG: null
++ - REPEATED_UTESTS_LONG_COUNT: 100
++ - REPEATED_UTESTS_STRESS: null
++ - REPEATED_UTESTS_STRESS_COUNT: 500
++ - REPEATED_JVM_DTESTS: null
++ - REPEATED_JVM_DTESTS_COUNT: 500
++ - REPEATED_JVM_UPGRADE_DTESTS: null
++ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
++ - REPEATED_DTESTS: null
++ - REPEATED_DTESTS_COUNT: 500
++ - REPEATED_UPGRADE_DTESTS: null
++ - REPEATED_UPGRADE_DTESTS_COUNT: 25
++ - REPEATED_ANT_TEST_TARGET: testsome
++ - REPEATED_ANT_TEST_CLASS: null
++ - REPEATED_ANT_TEST_METHODS: null
++ - REPEATED_ANT_TEST_COUNT: 500
++ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ j11_cqlsh_dtests_py3_vnode:
++ docker:
++ - image: apache/cassandra-testing-ubuntu2004-java11:latest
++ resource_class: medium
++ working_directory: ~/
++ shell: /bin/bash -eo pipefail -l
++ parallelism: 4
++ steps:
++ - attach_workspace:
++ at: /home/cassandra
++ - run:
++ name: Clone Cassandra dtest Repository (via git)
++ command: |
++ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
++ - run:
++ name: Configure virtualenv and python Dependencies
++ command: |
++ # note, this should be super quick as all dependencies should be pre-installed in the docker image
++ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
++ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
++ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
++ source ~/env3.6/bin/activate
++ export PATH=$JAVA_HOME/bin:$PATH
++ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
++ pip3 uninstall -y cqlsh
++ pip3 freeze
++ - run:
++ name: Determine Tests to Run (j11_with_vnodes)
++ no_output_timeout: 5m
++ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip [...]
++ - run:
++ name: Run dtests (j11_with_vnodes)
++ no_output_timeout: 15m
++ command: |
++ echo "cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt"
++ cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt
++
++ source ~/env3.6/bin/activate
++ export PATH=$JAVA_HOME/bin:$PATH
++ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then
++ export CQLSH_PYTHON=/usr/bin/python3.6
++ fi
++
++ java -version
++ cd ~/cassandra-dtest
++ mkdir -p /tmp/dtest
++
++ echo "env: $(env)"
++ echo "** done env"
++ mkdir -p /tmp/results/dtests
++ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
++ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt`
++ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
++ - store_test_results:
++ path: /tmp/results
++ - store_artifacts:
++ path: /tmp/dtest
++ destination: dtest_j11_with_vnodes
++ - store_artifacts:
++ path: ~/cassandra-dtest/logs
++ destination: dtest_j11_with_vnodes_logs
++ environment:
++ - ANT_HOME: /usr/share/ant
++ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ - LANG: en_US.UTF-8
++ - KEEP_TEST_DIR: true
++ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
++ - PYTHONIOENCODING: utf-8
++ - PYTHONUNBUFFERED: true
++ - CASS_DRIVER_NO_EXTENSIONS: true
++ - CASS_DRIVER_NO_CYTHON: true
++ - CASSANDRA_SKIP_SYNC: true
++ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
++ - DTEST_BRANCH: trunk
++ - CCM_MAX_HEAP_SIZE: 1024M
++ - CCM_HEAP_NEWSIZE: 256M
++ - REPEATED_TESTS_STOP_ON_FAILURE: false
++ - REPEATED_UTESTS: null
++ - REPEATED_UTESTS_COUNT: 500
++ - REPEATED_UTESTS_FQLTOOL: null
++ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
++ - REPEATED_UTESTS_LONG: null
++ - REPEATED_UTESTS_LONG_COUNT: 100
++ - REPEATED_UTESTS_STRESS: null
++ - REPEATED_UTESTS_STRESS_COUNT: 500
++ - REPEATED_JVM_DTESTS: null
++ - REPEATED_JVM_DTESTS_COUNT: 500
++ - REPEATED_JVM_UPGRADE_DTESTS: null
++ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
++ - REPEATED_DTESTS: null
++ - REPEATED_DTESTS_COUNT: 500
++ - REPEATED_UPGRADE_DTESTS: null
++ - REPEATED_UPGRADE_DTESTS_COUNT: 25
++ - REPEATED_ANT_TEST_TARGET: testsome
++ - REPEATED_ANT_TEST_CLASS: null
++ - REPEATED_ANT_TEST_METHODS: null
++ - REPEATED_ANT_TEST_COUNT: 500
++ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - CASSANDRA_USE_JDK11: true
++ j8_upgrade_dtests:
++ docker:
++ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
++ resource_class: medium
++ working_directory: ~/
++ shell: /bin/bash -eo pipefail -l
++ parallelism: 4
++ steps:
++ - attach_workspace:
++ at: /home/cassandra
++ - run:
++ name: Clone Cassandra dtest Repository (via git)
++ command: |
++ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
++ - run:
++ name: Configure virtualenv and python Dependencies
++ command: |
++ # note, this should be super quick as all dependencies should be pre-installed in the docker image
++ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
++ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
++ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
++ source ~/env3.6/bin/activate
++ export PATH=$JAVA_HOME/bin:$PATH
++ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
++ pip3 uninstall -y cqlsh
++ pip3 freeze
++ - run:
++ name: Determine Tests to Run (j8_upgradetests_without_vnodes)
++ no_output_timeout: 5m
++ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_upgradetests_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --ex [...]
++ - run:
++ name: Run dtests (j8_upgradetests_without_vnodes)
++ no_output_timeout: 15m
++ command: "echo \"cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually [...]
++ - store_test_results:
++ path: /tmp/results
++ - store_artifacts:
++ path: /tmp/dtest
++ destination: dtest_j8_upgradetests_without_vnodes
++ - store_artifacts:
++ path: ~/cassandra-dtest/logs
++ destination: dtest_j8_upgradetests_without_vnodes_logs
++ environment:
++ - ANT_HOME: /usr/share/ant
++ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ - LANG: en_US.UTF-8
++ - KEEP_TEST_DIR: true
++ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
++ - PYTHONIOENCODING: utf-8
++ - PYTHONUNBUFFERED: true
++ - CASS_DRIVER_NO_EXTENSIONS: true
++ - CASS_DRIVER_NO_CYTHON: true
++ - CASSANDRA_SKIP_SYNC: true
++ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
++ - DTEST_BRANCH: trunk
++ - CCM_MAX_HEAP_SIZE: 1024M
++ - CCM_HEAP_NEWSIZE: 256M
++ - REPEATED_TESTS_STOP_ON_FAILURE: false
++ - REPEATED_UTESTS: null
++ - REPEATED_UTESTS_COUNT: 500
++ - REPEATED_UTESTS_FQLTOOL: null
++ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
++ - REPEATED_UTESTS_LONG: null
++ - REPEATED_UTESTS_LONG_COUNT: 100
++ - REPEATED_UTESTS_STRESS: null
++ - REPEATED_UTESTS_STRESS_COUNT: 500
++ - REPEATED_JVM_DTESTS: null
++ - REPEATED_JVM_DTESTS_COUNT: 500
++ - REPEATED_JVM_UPGRADE_DTESTS: null
++ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
++ - REPEATED_DTESTS: null
++ - REPEATED_DTESTS_COUNT: 500
++ - REPEATED_UPGRADE_DTESTS: null
++ - REPEATED_UPGRADE_DTESTS_COUNT: 25
++ - REPEATED_ANT_TEST_TARGET: testsome
++ - REPEATED_ANT_TEST_CLASS: null
++ - REPEATED_ANT_TEST_METHODS: null
++ - REPEATED_ANT_TEST_COUNT: 500
++ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ j8_jvm_upgrade_dtests_repeat:
++ docker:
++ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
++ resource_class: medium
++ working_directory: ~/
++ shell: /bin/bash -eo pipefail -l
++ parallelism: 4
++ steps:
++ - attach_workspace:
++ at: /home/cassandra
++ - run:
++ name: Log Environment Information
++ command: |
++ echo '*** id ***'
++ id
++ echo '*** cat /proc/cpuinfo ***'
++ cat /proc/cpuinfo
++ echo '*** free -m ***'
++ free -m
++ echo '*** df -m ***'
++ df -m
++ echo '*** ifconfig -a ***'
++ ifconfig -a
++ echo '*** uname -a ***'
++ uname -a
++ echo '*** mount ***'
++ mount
++ echo '*** env ***'
++ env
++ echo '*** java ***'
++ which java
++ java -version
++ - run:
++ name: Repeatedly run new or modifed JUnit tests
++ no_output_timeout: 15m
++ command: |
++ set -x
++ export PATH=$JAVA_HOME/bin:$PATH
++ time mv ~/cassandra /tmp
++ cd /tmp/cassandra
++ if [ -d ~/dtest_jars ]; then
++ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
++ fi
++
++ # Calculate the number of test iterations to be run by the current parallel runner.
++ count=$((${REPEATED_JVM_UPGRADE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
++ if (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_UPGRADE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
++ count=$((count+1))
++ fi
++
++ # Put manually specified tests and automatically detected tests together, removing duplicates
++ tests=$(echo ${REPEATED_JVM_UPGRADE_DTESTS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
++ echo "Tests to be repeated: ${tests}"
++
++ # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
++ target=test-jvm-dtest-some
++ testtag=""
++ if [[ $target == "test-cdc" ]]; then
++ testtag="cdc"
++ elif [[ $target == "test-compression" ]]; then
++ testtag="compression"
++ elif [[ $target == "test-system-keyspace-directory" ]]; then
++ testtag="system_keyspace_directory"
++ fi
++
++ # Run each test class as many times as requested.
++ exit_code="$?"
++ for test in $tests; do
++
++ # Split class and method names from the test name
++ if [[ $test =~ "#" ]]; then
++ class=${test%"#"*}
++ method=${test#*"#"}
++ else
++ class=$test
++ method=""
++ fi
++
++ # Prepare the -Dtest.name argument.
++ # It can be the fully qualified class name or the short class name, depending on the target.
++ if [[ $target == "test" || \
++ $target == "test-cdc" || \
++ $target == "test-compression" || \
++ $target == "test-system-keyspace-directory" || \
++ $target == "fqltool-test" || \
++ $target == "long-test" || \
++ $target == "stress-test" ]]; then
++ name_arg="-Dtest.name=${class##*.}"
++ else
++ name_arg="-Dtest.name=$class"
++ fi
++
++ # Prepare the -Dtest.methods argument, which is optional
++ if [[ $method == "" ]]; then
++ methods_arg=""
++ else
++ methods_arg="-Dtest.methods=$method"
++ fi
++
++ for i in $(seq -w 1 $count); do
++ echo "Running test $test, iteration $i of $count"
++
++ # run the test
++ status="passes"
++ if !( set -o pipefail && \
++ ant test-jvm-dtest-some $name_arg $methods_arg -Dno-build-test=true | \
++ tee stdout.txt \
++ ); then
++ status="fails"
++ exit_code=1
++ fi
++
++ # move the stdout output file
++ dest=/tmp/results/repeated_utests/stdout/${status}/${i}
++ mkdir -p $dest
++ mv stdout.txt $dest/${test}.txt
++
++ # move the XML output files
++ source=build/test/output/${testtag}
++ dest=/tmp/results/repeated_utests/output/${status}/${i}
++ mkdir -p $dest
++ if [[ -d $source && -n "$(ls $source)" ]]; then
++ mv $source/* $dest/
++ fi
++
++ # move the log files
++ source=build/test/logs/${testtag}
++ dest=/tmp/results/repeated_utests/logs/${status}/${i}
++ mkdir -p $dest
++ if [[ -d $source && -n "$(ls $source)" ]]; then
++ mv $source/* $dest/
++ fi
++
++ # maybe stop iterations on test failure
++ if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
++ break
++ fi
++ done
++ done
++ (exit ${exit_code})
++ - store_test_results:
++ path: /tmp/results/repeated_utests/output
++ - store_artifacts:
++ path: /tmp/results/repeated_utests/stdout
++ destination: stdout
++ - store_artifacts:
++ path: /tmp/results/repeated_utests/output
++ destination: junitxml
++ - store_artifacts:
++ path: /tmp/results/repeated_utests/logs
++ destination: logs
++ environment:
++ - ANT_HOME: /usr/share/ant
++ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ - LANG: en_US.UTF-8
++ - KEEP_TEST_DIR: true
++ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
++ - PYTHONIOENCODING: utf-8
++ - PYTHONUNBUFFERED: true
++ - CASS_DRIVER_NO_EXTENSIONS: true
++ - CASS_DRIVER_NO_CYTHON: true
++ - CASSANDRA_SKIP_SYNC: true
++ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
++ - DTEST_BRANCH: trunk
++ - CCM_MAX_HEAP_SIZE: 1024M
++ - CCM_HEAP_NEWSIZE: 256M
++ - REPEATED_TESTS_STOP_ON_FAILURE: false
++ - REPEATED_UTESTS: null
++ - REPEATED_UTESTS_COUNT: 500
++ - REPEATED_UTESTS_FQLTOOL: null
++ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
++ - REPEATED_UTESTS_LONG: null
++ - REPEATED_UTESTS_LONG_COUNT: 100
++ - REPEATED_UTESTS_STRESS: null
++ - REPEATED_UTESTS_STRESS_COUNT: 500
++ - REPEATED_JVM_DTESTS: null
++ - REPEATED_JVM_DTESTS_COUNT: 500
++ - REPEATED_JVM_UPGRADE_DTESTS: null
++ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
++ - REPEATED_DTESTS: null
++ - REPEATED_DTESTS_COUNT: 500
++ - REPEATED_UPGRADE_DTESTS: null
++ - REPEATED_UPGRADE_DTESTS_COUNT: 25
++ - REPEATED_ANT_TEST_TARGET: testsome
++ - REPEATED_ANT_TEST_CLASS: null
++ - REPEATED_ANT_TEST_METHODS: null
++ - REPEATED_ANT_TEST_COUNT: 500
++ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ j11_cqlsh_dtests_py38_vnode:
++ docker:
++ - image: apache/cassandra-testing-ubuntu2004-java11:latest
++ resource_class: medium
++ working_directory: ~/
++ shell: /bin/bash -eo pipefail -l
++ parallelism: 4
++ steps:
++ - attach_workspace:
++ at: /home/cassandra
++ - run:
++ name: Clone Cassandra dtest Repository (via git)
++ command: |
++ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
++ - run:
++ name: Configure virtualenv and python Dependencies
++ command: |
++ # note, this should be super quick as all dependencies should be pre-installed in the docker image
++ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
++ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
++ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
++ source ~/env3.8/bin/activate
++ export PATH=$JAVA_HOME/bin:$PATH
++ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
++ pip3 uninstall -y cqlsh
++ pip3 freeze
++ - run:
++ name: Determine Tests to Run (j11_with_vnodes)
++ no_output_timeout: 5m
++ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip [...]
++ - run:
++ name: Run dtests (j11_with_vnodes)
++ no_output_timeout: 15m
++ command: |
++ echo "cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt"
++ cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt
++
++ source ~/env3.8/bin/activate
++ export PATH=$JAVA_HOME/bin:$PATH
++ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then
++ export CQLSH_PYTHON=/usr/bin/python3.8
++ fi
++
++ java -version
++ cd ~/cassandra-dtest
++ mkdir -p /tmp/dtest
++
++ echo "env: $(env)"
++ echo "** done env"
++ mkdir -p /tmp/results/dtests
++ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
++ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt`
++ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
++ - store_test_results:
++ path: /tmp/results
++ - store_artifacts:
++ path: /tmp/dtest
++ destination: dtest_j11_with_vnodes
++ - store_artifacts:
++ path: ~/cassandra-dtest/logs
++ destination: dtest_j11_with_vnodes_logs
++ environment:
++ - ANT_HOME: /usr/share/ant
++ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ - LANG: en_US.UTF-8
++ - KEEP_TEST_DIR: true
++ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
++ - PYTHONIOENCODING: utf-8
++ - PYTHONUNBUFFERED: true
++ - CASS_DRIVER_NO_EXTENSIONS: true
++ - CASS_DRIVER_NO_CYTHON: true
++ - CASSANDRA_SKIP_SYNC: true
++ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
++ - DTEST_BRANCH: trunk
++ - CCM_MAX_HEAP_SIZE: 1024M
++ - CCM_HEAP_NEWSIZE: 256M
++ - REPEATED_TESTS_STOP_ON_FAILURE: false
++ - REPEATED_UTESTS: null
++ - REPEATED_UTESTS_COUNT: 500
++ - REPEATED_UTESTS_FQLTOOL: null
++ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
++ - REPEATED_UTESTS_LONG: null
++ - REPEATED_UTESTS_LONG_COUNT: 100
++ - REPEATED_UTESTS_STRESS: null
++ - REPEATED_UTESTS_STRESS_COUNT: 500
++ - REPEATED_JVM_DTESTS: null
++ - REPEATED_JVM_DTESTS_COUNT: 500
++ - REPEATED_JVM_UPGRADE_DTESTS: null
++ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
++ - REPEATED_DTESTS: null
++ - REPEATED_DTESTS_COUNT: 500
++ - REPEATED_UPGRADE_DTESTS: null
++ - REPEATED_UPGRADE_DTESTS_COUNT: 25
++ - REPEATED_ANT_TEST_TARGET: testsome
++ - REPEATED_ANT_TEST_CLASS: null
++ - REPEATED_ANT_TEST_METHODS: null
++ - REPEATED_ANT_TEST_COUNT: 500
++ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - CASSANDRA_USE_JDK11: true
++ j11_jvm_dtests_repeat:
++ docker:
++ - image: apache/cassandra-testing-ubuntu2004-java11:latest
++ resource_class: medium
++ working_directory: ~/
++ shell: /bin/bash -eo pipefail -l
++ parallelism: 4
++ steps:
++ - attach_workspace:
++ at: /home/cassandra
++ - run:
++ name: Log Environment Information
++ command: |
++ echo '*** id ***'
++ id
++ echo '*** cat /proc/cpuinfo ***'
++ cat /proc/cpuinfo
++ echo '*** free -m ***'
++ free -m
++ echo '*** df -m ***'
++ df -m
++ echo '*** ifconfig -a ***'
++ ifconfig -a
++ echo '*** uname -a ***'
++ uname -a
++ echo '*** mount ***'
++ mount
++ echo '*** env ***'
++ env
++ echo '*** java ***'
++ which java
++ java -version
++ - run:
++ name: Repeatedly run new or modifed JUnit tests
++ no_output_timeout: 15m
++ command: |
++ set -x
++ export PATH=$JAVA_HOME/bin:$PATH
++ time mv ~/cassandra /tmp
++ cd /tmp/cassandra
++ if [ -d ~/dtest_jars ]; then
++ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
++ fi
++
++ # Calculate the number of test iterations to be run by the current parallel runner.
++ count=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
++ if (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
++ count=$((count+1))
++ fi
++
++ # Put manually specified tests and automatically detected tests together, removing duplicates
++ tests=$(echo ${REPEATED_JVM_DTESTS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
++ echo "Tests to be repeated: ${tests}"
++
++ # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
++ target=test-jvm-dtest-some
++ testtag=""
++ if [[ $target == "test-cdc" ]]; then
++ testtag="cdc"
++ elif [[ $target == "test-compression" ]]; then
++ testtag="compression"
++ elif [[ $target == "test-system-keyspace-directory" ]]; then
++ testtag="system_keyspace_directory"
++ fi
++
++ # Run each test class as many times as requested.
++ exit_code="$?"
++ for test in $tests; do
++
++ # Split class and method names from the test name
++ if [[ $test =~ "#" ]]; then
++ class=${test%"#"*}
++ method=${test#*"#"}
++ else
++ class=$test
++ method=""
++ fi
++
++ # Prepare the -Dtest.name argument.
++ # It can be the fully qualified class name or the short class name, depending on the target.
++ if [[ $target == "test" || \
++ $target == "test-cdc" || \
++ $target == "test-compression" || \
++ $target == "test-system-keyspace-directory" || \
++ $target == "fqltool-test" || \
++ $target == "long-test" || \
++ $target == "stress-test" ]]; then
++ name_arg="-Dtest.name=${class##*.}"
++ else
++ name_arg="-Dtest.name=$class"
++ fi
++
++ # Prepare the -Dtest.methods argument, which is optional
++ if [[ $method == "" ]]; then
++ methods_arg=""
++ else
++ methods_arg="-Dtest.methods=$method"
++ fi
++
++ for i in $(seq -w 1 $count); do
++ echo "Running test $test, iteration $i of $count"
++
++ # run the test
++ status="passes"
++ if !( set -o pipefail && \
++ ant test-jvm-dtest-some $name_arg $methods_arg -Dno-build-test=true | \
++ tee stdout.txt \
++ ); then
++ status="fails"
++ exit_code=1
++ fi
++
++ # move the stdout output file
++ dest=/tmp/results/repeated_utests/stdout/${status}/${i}
++ mkdir -p $dest
++ mv stdout.txt $dest/${test}.txt
++
++ # move the XML output files
++ source=build/test/output/${testtag}
++ dest=/tmp/results/repeated_utests/output/${status}/${i}
++ mkdir -p $dest
++ if [[ -d $source && -n "$(ls $source)" ]]; then
++ mv $source/* $dest/
++ fi
++
++ # move the log files
++ source=build/test/logs/${testtag}
++ dest=/tmp/results/repeated_utests/logs/${status}/${i}
++ mkdir -p $dest
++ if [[ -d $source && -n "$(ls $source)" ]]; then
++ mv $source/* $dest/
++ fi
++
++ # maybe stop iterations on test failure
++ if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
++ break
++ fi
++ done
++ done
++ (exit ${exit_code})
++ - store_test_results:
++ path: /tmp/results/repeated_utests/output
++ - store_artifacts:
++ path: /tmp/results/repeated_utests/stdout
++ destination: stdout
++ - store_artifacts:
++ path: /tmp/results/repeated_utests/output
++ destination: junitxml
++ - store_artifacts:
++ path: /tmp/results/repeated_utests/logs
++ destination: logs
++ environment:
++ - ANT_HOME: /usr/share/ant
++ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ - LANG: en_US.UTF-8
++ - KEEP_TEST_DIR: true
++ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
++ - PYTHONIOENCODING: utf-8
++ - PYTHONUNBUFFERED: true
++ - CASS_DRIVER_NO_EXTENSIONS: true
++ - CASS_DRIVER_NO_CYTHON: true
++ - CASSANDRA_SKIP_SYNC: true
++ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
++ - DTEST_BRANCH: trunk
++ - CCM_MAX_HEAP_SIZE: 1024M
++ - CCM_HEAP_NEWSIZE: 256M
++ - REPEATED_TESTS_STOP_ON_FAILURE: false
++ - REPEATED_UTESTS: null
++ - REPEATED_UTESTS_COUNT: 500
++ - REPEATED_UTESTS_FQLTOOL: null
++ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
++ - REPEATED_UTESTS_LONG: null
++ - REPEATED_UTESTS_LONG_COUNT: 100
++ - REPEATED_UTESTS_STRESS: null
++ - REPEATED_UTESTS_STRESS_COUNT: 500
++ - REPEATED_JVM_DTESTS: null
++ - REPEATED_JVM_DTESTS_COUNT: 500
++ - REPEATED_JVM_UPGRADE_DTESTS: null
++ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
++ - REPEATED_DTESTS: null
++ - REPEATED_DTESTS_COUNT: 500
++ - REPEATED_UPGRADE_DTESTS: null
++ - REPEATED_UPGRADE_DTESTS_COUNT: 25
++ - REPEATED_ANT_TEST_TARGET: testsome
++ - REPEATED_ANT_TEST_CLASS: null
++ - REPEATED_ANT_TEST_METHODS: null
++ - REPEATED_ANT_TEST_COUNT: 500
++ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - CASSANDRA_USE_JDK11: true
++ j8_utests_long_repeat:
++ docker:
++ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
++ resource_class: medium
++ working_directory: ~/
++ shell: /bin/bash -eo pipefail -l
++ parallelism: 4
++ steps:
++ - attach_workspace:
++ at: /home/cassandra
++ - run:
++ name: Log Environment Information
++ command: |
++ echo '*** id ***'
++ id
++ echo '*** cat /proc/cpuinfo ***'
++ cat /proc/cpuinfo
++ echo '*** free -m ***'
++ free -m
++ echo '*** df -m ***'
++ df -m
++ echo '*** ifconfig -a ***'
++ ifconfig -a
++ echo '*** uname -a ***'
++ uname -a
++ echo '*** mount ***'
++ mount
++ echo '*** env ***'
++ env
++ echo '*** java ***'
++ which java
++ java -version
++ - run:
++ name: Repeatedly run new or modifed JUnit tests
++ no_output_timeout: 15m
++ command: |
++ set -x
++ export PATH=$JAVA_HOME/bin:$PATH
++ time mv ~/cassandra /tmp
++ cd /tmp/cassandra
++ if [ -d ~/dtest_jars ]; then
++ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
++ fi
++
++ # Calculate the number of test iterations to be run by the current parallel runner.
++ count=$((${REPEATED_UTESTS_LONG_COUNT} / CIRCLE_NODE_TOTAL))
++ if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_LONG_COUNT} % CIRCLE_NODE_TOTAL))); then
++ count=$((count+1))
++ fi
++
++ # Put manually specified tests and automatically detected tests together, removing duplicates
++ tests=$(echo ${REPEATED_UTESTS_LONG} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
++ echo "Tests to be repeated: ${tests}"
++
++ # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
++ target=long-testsome
++ testtag=""
++ if [[ $target == "test-cdc" ]]; then
++ testtag="cdc"
++ elif [[ $target == "test-compression" ]]; then
++ testtag="compression"
++ elif [[ $target == "test-system-keyspace-directory" ]]; then
++ testtag="system_keyspace_directory"
++ fi
++
++ # Run each test class as many times as requested.
++ exit_code="$?"
++ for test in $tests; do
++
++ # Split class and method names from the test name
++ if [[ $test =~ "#" ]]; then
++ class=${test%"#"*}
++ method=${test#*"#"}
++ else
++ class=$test
++ method=""
++ fi
++
++ # Prepare the -Dtest.name argument.
++ # It can be the fully qualified class name or the short class name, depending on the target.
++ if [[ $target == "test" || \
++ $target == "test-cdc" || \
++ $target == "test-compression" || \
++ $target == "test-system-keyspace-directory" || \
++ $target == "fqltool-test" || \
++ $target == "long-test" || \
++ $target == "stress-test" ]]; then
++ name_arg="-Dtest.name=${class##*.}"
++ else
++ name_arg="-Dtest.name=$class"
++ fi
++
++ # Prepare the -Dtest.methods argument, which is optional
++ if [[ $method == "" ]]; then
++ methods_arg=""
++ else
++ methods_arg="-Dtest.methods=$method"
++ fi
++
++ for i in $(seq -w 1 $count); do
++ echo "Running test $test, iteration $i of $count"
++
++ # run the test
++ status="passes"
++ if !( set -o pipefail && \
++ ant long-testsome $name_arg $methods_arg -Dno-build-test=true | \
++ tee stdout.txt \
++ ); then
++ status="fails"
++ exit_code=1
++ fi
++
++ # move the stdout output file
++ dest=/tmp/results/repeated_utests/stdout/${status}/${i}
++ mkdir -p $dest
++ mv stdout.txt $dest/${test}.txt
++
++ # move the XML output files
++ source=build/test/output/${testtag}
++ dest=/tmp/results/repeated_utests/output/${status}/${i}
++ mkdir -p $dest
++ if [[ -d $source && -n "$(ls $source)" ]]; then
++ mv $source/* $dest/
++ fi
++
++ # move the log files
++ source=build/test/logs/${testtag}
++ dest=/tmp/results/repeated_utests/logs/${status}/${i}
++ mkdir -p $dest
++ if [[ -d $source && -n "$(ls $source)" ]]; then
++ mv $source/* $dest/
++ fi
++
++ # maybe stop iterations on test failure
++ if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
++ break
++ fi
++ done
++ done
++ (exit ${exit_code})
++ - store_test_results:
++ path: /tmp/results/repeated_utests/output
++ - store_artifacts:
++ path: /tmp/results/repeated_utests/stdout
++ destination: stdout
++ - store_artifacts:
++ path: /tmp/results/repeated_utests/output
++ destination: junitxml
++ - store_artifacts:
++ path: /tmp/results/repeated_utests/logs
++ destination: logs
++ environment:
++ - ANT_HOME: /usr/share/ant
++ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ - LANG: en_US.UTF-8
++ - KEEP_TEST_DIR: true
++ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
++ - PYTHONIOENCODING: utf-8
++ - PYTHONUNBUFFERED: true
++ - CASS_DRIVER_NO_EXTENSIONS: true
++ - CASS_DRIVER_NO_CYTHON: true
++ - CASSANDRA_SKIP_SYNC: true
++ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
++ - DTEST_BRANCH: trunk
++ - CCM_MAX_HEAP_SIZE: 1024M
++ - CCM_HEAP_NEWSIZE: 256M
++ - REPEATED_TESTS_STOP_ON_FAILURE: false
++ - REPEATED_UTESTS: null
++ - REPEATED_UTESTS_COUNT: 500
++ - REPEATED_UTESTS_FQLTOOL: null
++ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
++ - REPEATED_UTESTS_LONG: null
++ - REPEATED_UTESTS_LONG_COUNT: 100
++ - REPEATED_UTESTS_STRESS: null
++ - REPEATED_UTESTS_STRESS_COUNT: 500
++ - REPEATED_JVM_DTESTS: null
++ - REPEATED_JVM_DTESTS_COUNT: 500
++ - REPEATED_JVM_UPGRADE_DTESTS: null
++ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
++ - REPEATED_DTESTS: null
++ - REPEATED_DTESTS_COUNT: 500
++ - REPEATED_UPGRADE_DTESTS: null
++ - REPEATED_UPGRADE_DTESTS_COUNT: 25
++ - REPEATED_ANT_TEST_TARGET: testsome
++ - REPEATED_ANT_TEST_CLASS: null
++ - REPEATED_ANT_TEST_METHODS: null
++ - REPEATED_ANT_TEST_COUNT: 500
++ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ j8_unit_tests:
++ docker:
++ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
++ resource_class: medium
++ working_directory: ~/
++ shell: /bin/bash -eo pipefail -l
++ parallelism: 4
++ steps:
++ - attach_workspace:
++ at: /home/cassandra
++ - run:
++ name: Determine unit Tests to Run
++ command: |
++ # reminder: this code (along with all the steps) is independently executed on every circle container
++ # so the goal here is to get the circleci script to return the tests *this* container will run
++ # which we do via the `circleci` cli tool.
++
++ rm -fr ~/cassandra-dtest/upgrade_tests
++ echo "***java tests***"
++
++ # get all of our unit test filenames
++ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
++
++ # split up the unit tests into groups based on the number of containers we have
++ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
++ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
++ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
++ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
++ no_output_timeout: 15m
++ - run:
++ name: Log Environment Information
++ command: |
++ echo '*** id ***'
++ id
++ echo '*** cat /proc/cpuinfo ***'
++ cat /proc/cpuinfo
++ echo '*** free -m ***'
++ free -m
++ echo '*** df -m ***'
++ df -m
++ echo '*** ifconfig -a ***'
++ ifconfig -a
++ echo '*** uname -a ***'
++ uname -a
++ echo '*** mount ***'
++ mount
++ echo '*** env ***'
++ env
++ echo '*** java ***'
++ which java
++ java -version
++ - run:
++ name: Run Unit Tests (testclasslist)
++ command: |
++ set -x
++ export PATH=$JAVA_HOME/bin:$PATH
++ time mv ~/cassandra /tmp
++ cd /tmp/cassandra
++ if [ -d ~/dtest_jars ]; then
++ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
++ fi
++ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
++ if [ -z "$test_timeout" ]; then
++ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
++ fi
++ ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
++ no_output_timeout: 15m
++ - store_test_results:
++ path: /tmp/cassandra/build/test/output/
++ - store_artifacts:
++ path: /tmp/cassandra/build/test/output
++ destination: junitxml
++ - store_artifacts:
++ path: /tmp/cassandra/build/test/logs
++ destination: logs
++ environment:
++ - ANT_HOME: /usr/share/ant
++ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ - LANG: en_US.UTF-8
++ - KEEP_TEST_DIR: true
++ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
++ - PYTHONIOENCODING: utf-8
++ - PYTHONUNBUFFERED: true
++ - CASS_DRIVER_NO_EXTENSIONS: true
++ - CASS_DRIVER_NO_CYTHON: true
++ - CASSANDRA_SKIP_SYNC: true
++ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
++ - DTEST_BRANCH: trunk
++ - CCM_MAX_HEAP_SIZE: 1024M
++ - CCM_HEAP_NEWSIZE: 256M
++ - REPEATED_TESTS_STOP_ON_FAILURE: false
++ - REPEATED_UTESTS: null
++ - REPEATED_UTESTS_COUNT: 500
++ - REPEATED_UTESTS_FQLTOOL: null
++ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
++ - REPEATED_UTESTS_LONG: null
++ - REPEATED_UTESTS_LONG_COUNT: 100
++ - REPEATED_UTESTS_STRESS: null
++ - REPEATED_UTESTS_STRESS_COUNT: 500
++ - REPEATED_JVM_DTESTS: null
++ - REPEATED_JVM_DTESTS_COUNT: 500
++ - REPEATED_JVM_UPGRADE_DTESTS: null
++ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
++ - REPEATED_DTESTS: null
++ - REPEATED_DTESTS_COUNT: 500
++ - REPEATED_UPGRADE_DTESTS: null
++ - REPEATED_UPGRADE_DTESTS_COUNT: 25
++ - REPEATED_ANT_TEST_TARGET: testsome
++ - REPEATED_ANT_TEST_CLASS: null
++ - REPEATED_ANT_TEST_METHODS: null
++ - REPEATED_ANT_TEST_COUNT: 500
++ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ j11_jvm_dtests:
++ docker:
++ - image: apache/cassandra-testing-ubuntu2004-java11:latest
++ resource_class: medium
++ working_directory: ~/
++ shell: /bin/bash -eo pipefail -l
++ parallelism: 1
++ steps:
++ - attach_workspace:
++ at: /home/cassandra
++ - run:
++ name: Determine distributed Tests to Run
++ command: |
++ # reminder: this code (along with all the steps) is independently executed on every circle container
++ # so the goal here is to get the circleci script to return the tests *this* container will run
++ # which we do via the `circleci` cli tool.
++
++ rm -fr ~/cassandra-dtest/upgrade_tests
++ echo "***java tests***"
++
++ # get all of our unit test filenames
++ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt
++
++ # split up the unit tests into groups based on the number of containers we have
++ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
++ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep -v upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
++ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
++ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
++ no_output_timeout: 15m
++ - run:
++ name: Log Environment Information
++ command: |
++ echo '*** id ***'
++ id
++ echo '*** cat /proc/cpuinfo ***'
++ cat /proc/cpuinfo
++ echo '*** free -m ***'
++ free -m
++ echo '*** df -m ***'
++ df -m
++ echo '*** ifconfig -a ***'
++ ifconfig -a
++ echo '*** uname -a ***'
++ uname -a
++ echo '*** mount ***'
++ mount
++ echo '*** env ***'
++ env
++ echo '*** java ***'
++ which java
++ java -version
++ - run:
++ name: Run Unit Tests (testclasslist)
++ command: |
++ set -x
++ export PATH=$JAVA_HOME/bin:$PATH
++ time mv ~/cassandra /tmp
++ cd /tmp/cassandra
++ if [ -d ~/dtest_jars ]; then
++ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
++ fi
++ test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true)
++ if [ -z "$test_timeout" ]; then
++ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
++ fi
++ ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed
++ no_output_timeout: 15m
++ - store_test_results:
++ path: /tmp/cassandra/build/test/output/
++ - store_artifacts:
++ path: /tmp/cassandra/build/test/output
++ destination: junitxml
++ - store_artifacts:
++ path: /tmp/cassandra/build/test/logs
++ destination: logs
++ environment:
++ - ANT_HOME: /usr/share/ant
++ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ - LANG: en_US.UTF-8
++ - KEEP_TEST_DIR: true
++ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
++ - PYTHONIOENCODING: utf-8
++ - PYTHONUNBUFFERED: true
++ - CASS_DRIVER_NO_EXTENSIONS: true
++ - CASS_DRIVER_NO_CYTHON: true
++ - CASSANDRA_SKIP_SYNC: true
++ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
++ - DTEST_BRANCH: trunk
++ - CCM_MAX_HEAP_SIZE: 1024M
++ - CCM_HEAP_NEWSIZE: 256M
++ - REPEATED_TESTS_STOP_ON_FAILURE: false
++ - REPEATED_UTESTS: null
++ - REPEATED_UTESTS_COUNT: 500
++ - REPEATED_UTESTS_FQLTOOL: null
++ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
++ - REPEATED_UTESTS_LONG: null
++ - REPEATED_UTESTS_LONG_COUNT: 100
++ - REPEATED_UTESTS_STRESS: null
++ - REPEATED_UTESTS_STRESS_COUNT: 500
++ - REPEATED_JVM_DTESTS: null
++ - REPEATED_JVM_DTESTS_COUNT: 500
++ - REPEATED_JVM_UPGRADE_DTESTS: null
++ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
++ - REPEATED_DTESTS: null
++ - REPEATED_DTESTS_COUNT: 500
++ - REPEATED_UPGRADE_DTESTS: null
++ - REPEATED_UPGRADE_DTESTS_COUNT: 25
++ - REPEATED_ANT_TEST_TARGET: testsome
++ - REPEATED_ANT_TEST_CLASS: null
++ - REPEATED_ANT_TEST_METHODS: null
++ - REPEATED_ANT_TEST_COUNT: 500
++ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - CASSANDRA_USE_JDK11: true
++ j11_build:
++ docker:
++ - image: apache/cassandra-testing-ubuntu2004-java11:latest
++ resource_class: medium
++ working_directory: ~/
++ shell: /bin/bash -eo pipefail -l
++ parallelism: 1
++ steps:
++ - run:
++ name: Log Environment Information
++ command: |
++ echo '*** id ***'
++ id
++ echo '*** cat /proc/cpuinfo ***'
++ cat /proc/cpuinfo
++ echo '*** free -m ***'
++ free -m
++ echo '*** df -m ***'
++ df -m
++ echo '*** ifconfig -a ***'
++ ifconfig -a
++ echo '*** uname -a ***'
++ uname -a
++ echo '*** mount ***'
++ mount
++ echo '*** env ***'
++ env
++ echo '*** java ***'
++ which java
++ java -version
++ - run:
++ name: Clone Cassandra Repository (via git)
++ command: |
++ git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH https://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra
++ - run:
++ name: Build Cassandra
++ command: |
++ export PATH=$JAVA_HOME/bin:$PATH
++ cd ~/cassandra
++ # Loop to prevent failure due to maven-ant-tasks not downloading a jar..
++ for x in $(seq 1 3); do
++ ${ANT_HOME}/bin/ant clean realclean jar
++ RETURN="$?"
++ if [ "${RETURN}" -eq "0" ]; then
++ break
++ fi
++ done
++ # Exit, if we didn't build successfully
++ if [ "${RETURN}" -ne "0" ]; then
++ echo "Build failed with exit code: ${RETURN}"
++ exit ${RETURN}
++ fi
++ no_output_timeout: 15m
++ - run:
++ name: Run eclipse-warnings
++ command: |
++ export PATH=$JAVA_HOME/bin:$PATH
++ cd ~/cassandra
++ ant eclipse-warnings
++ - persist_to_workspace:
++ root: /home/cassandra
++ paths:
++ - cassandra
++ - .m2
++ environment:
++ - ANT_HOME: /usr/share/ant
++ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ - LANG: en_US.UTF-8
++ - KEEP_TEST_DIR: true
++ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
++ - PYTHONIOENCODING: utf-8
++ - PYTHONUNBUFFERED: true
++ - CASS_DRIVER_NO_EXTENSIONS: true
++ - CASS_DRIVER_NO_CYTHON: true
++ - CASSANDRA_SKIP_SYNC: true
++ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
++ - DTEST_BRANCH: trunk
++ - CCM_MAX_HEAP_SIZE: 1024M
++ - CCM_HEAP_NEWSIZE: 256M
++ - REPEATED_TESTS_STOP_ON_FAILURE: false
++ - REPEATED_UTESTS: null
++ - REPEATED_UTESTS_COUNT: 500
++ - REPEATED_UTESTS_FQLTOOL: null
++ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
++ - REPEATED_UTESTS_LONG: null
++ - REPEATED_UTESTS_LONG_COUNT: 100
++ - REPEATED_UTESTS_STRESS: null
++ - REPEATED_UTESTS_STRESS_COUNT: 500
++ - REPEATED_JVM_DTESTS: null
++ - REPEATED_JVM_DTESTS_COUNT: 500
++ - REPEATED_JVM_UPGRADE_DTESTS: null
++ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
++ - REPEATED_DTESTS: null
++ - REPEATED_DTESTS_COUNT: 500
++ - REPEATED_UPGRADE_DTESTS: null
++ - REPEATED_UPGRADE_DTESTS_COUNT: 25
++ - REPEATED_ANT_TEST_TARGET: testsome
++ - REPEATED_ANT_TEST_CLASS: null
++ - REPEATED_ANT_TEST_METHODS: null
++ - REPEATED_ANT_TEST_COUNT: 500
++ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - CASSANDRA_USE_JDK11: true
++ j8_dtests:
++ docker:
++ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
++ resource_class: medium
++ working_directory: ~/
++ shell: /bin/bash -eo pipefail -l
++ parallelism: 4
++ steps:
++ - attach_workspace:
++ at: /home/cassandra
++ - run:
++ name: Clone Cassandra dtest Repository (via git)
++ command: |
++ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
++ - run:
++ name: Configure virtualenv and python Dependencies
++ command: |
++ # note, this should be super quick as all dependencies should be pre-installed in the docker image
++ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
++ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
++ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
++ source ~/env3.6/bin/activate
++ export PATH=$JAVA_HOME/bin:$PATH
++ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
++ pip3 uninstall -y cqlsh
++ pip3 freeze
++ - run:
++ name: Determine Tests to Run (j8_without_vnodes)
++ no_output_timeout: 5m
++ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-i [...]
++ - run:
++ name: Run dtests (j8_without_vnodes)
++ no_output_timeout: 15m
++ command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not [...]
++ - store_test_results:
++ path: /tmp/results
++ - store_artifacts:
++ path: /tmp/dtest
++ destination: dtest_j8_without_vnodes
++ - store_artifacts:
++ path: ~/cassandra-dtest/logs
++ destination: dtest_j8_without_vnodes_logs
++ environment:
++ - ANT_HOME: /usr/share/ant
++ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ - LANG: en_US.UTF-8
++ - KEEP_TEST_DIR: true
++ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
++ - PYTHONIOENCODING: utf-8
++ - PYTHONUNBUFFERED: true
++ - CASS_DRIVER_NO_EXTENSIONS: true
++ - CASS_DRIVER_NO_CYTHON: true
++ - CASSANDRA_SKIP_SYNC: true
++ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
++ - DTEST_BRANCH: trunk
++ - CCM_MAX_HEAP_SIZE: 1024M
++ - CCM_HEAP_NEWSIZE: 256M
++ - REPEATED_TESTS_STOP_ON_FAILURE: false
++ - REPEATED_UTESTS: null
++ - REPEATED_UTESTS_COUNT: 500
++ - REPEATED_UTESTS_FQLTOOL: null
++ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
++ - REPEATED_UTESTS_LONG: null
++ - REPEATED_UTESTS_LONG_COUNT: 100
++ - REPEATED_UTESTS_STRESS: null
++ - REPEATED_UTESTS_STRESS_COUNT: 500
++ - REPEATED_JVM_DTESTS: null
++ - REPEATED_JVM_DTESTS_COUNT: 500
++ - REPEATED_JVM_UPGRADE_DTESTS: null
++ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
++ - REPEATED_DTESTS: null
++ - REPEATED_DTESTS_COUNT: 500
++ - REPEATED_UPGRADE_DTESTS: null
++ - REPEATED_UPGRADE_DTESTS_COUNT: 25
++ - REPEATED_ANT_TEST_TARGET: testsome
++ - REPEATED_ANT_TEST_CLASS: null
++ - REPEATED_ANT_TEST_METHODS: null
++ - REPEATED_ANT_TEST_COUNT: 500
++ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ j11_cqlsh-dtests-py2-no-vnodes:
++ docker:
++ - image: apache/cassandra-testing-ubuntu2004-java11:latest
++ resource_class: medium
++ working_directory: ~/
++ shell: /bin/bash -eo pipefail -l
++ parallelism: 4
++ steps:
++ - attach_workspace:
++ at: /home/cassandra
++ - run:
++ name: Clone Cassandra dtest Repository (via git)
++ command: |
++ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
++ - run:
++ name: Configure virtualenv and python Dependencies
++ command: |
++ # note, this should be super quick as all dependencies should be pre-installed in the docker image
++ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
++ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
++ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
++ source ~/env3.6/bin/activate
++ export PATH=$JAVA_HOME/bin:$PATH
++ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
++ pip3 uninstall -y cqlsh
++ pip3 freeze
++ - run:
++ name: Determine Tests to Run (j11_without_vnodes)
++ no_output_timeout: 5m
++ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource- [...]
++ - run:
++ name: Run dtests (j11_without_vnodes)
++ no_output_timeout: 15m
++ command: |
++ echo "cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt"
++ cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt
++
++ source ~/env3.6/bin/activate
++ export PATH=$JAVA_HOME/bin:$PATH
++ if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then
++ export CQLSH_PYTHON=/usr/bin/python2.7
++ fi
+
- if (($count <= 0)); then
- echo "No tests to run in this runner"
- else
- echo "Running ${REPEATED_DTESTS} $count times"
++ java -version
++ cd ~/cassandra-dtest
++ mkdir -p /tmp/dtest
+
- source ~/env3.6/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
++ echo "env: $(env)"
++ echo "** done env"
++ mkdir -p /tmp/results/dtests
++ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
++ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt`
++ set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
++ - store_test_results:
++ path: /tmp/results
++ - store_artifacts:
++ path: /tmp/dtest
++ destination: dtest_j11_without_vnodes
++ - store_artifacts:
++ path: ~/cassandra-dtest/logs
++ destination: dtest_j11_without_vnodes_logs
++ environment:
++ - ANT_HOME: /usr/share/ant
++ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ - LANG: en_US.UTF-8
++ - KEEP_TEST_DIR: true
++ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
++ - PYTHONIOENCODING: utf-8
++ - PYTHONUNBUFFERED: true
++ - CASS_DRIVER_NO_EXTENSIONS: true
++ - CASS_DRIVER_NO_CYTHON: true
++ - CASSANDRA_SKIP_SYNC: true
++ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
++ - DTEST_BRANCH: trunk
++ - CCM_MAX_HEAP_SIZE: 1024M
++ - CCM_HEAP_NEWSIZE: 256M
++ - REPEATED_TESTS_STOP_ON_FAILURE: false
++ - REPEATED_UTESTS: null
++ - REPEATED_UTESTS_COUNT: 500
++ - REPEATED_UTESTS_FQLTOOL: null
++ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
++ - REPEATED_UTESTS_LONG: null
++ - REPEATED_UTESTS_LONG_COUNT: 100
++ - REPEATED_UTESTS_STRESS: null
++ - REPEATED_UTESTS_STRESS_COUNT: 500
++ - REPEATED_JVM_DTESTS: null
++ - REPEATED_JVM_DTESTS_COUNT: 500
++ - REPEATED_JVM_UPGRADE_DTESTS: null
++ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
++ - REPEATED_DTESTS: null
++ - REPEATED_DTESTS_COUNT: 500
++ - REPEATED_UPGRADE_DTESTS: null
++ - REPEATED_UPGRADE_DTESTS_COUNT: 25
++ - REPEATED_ANT_TEST_TARGET: testsome
++ - REPEATED_ANT_TEST_CLASS: null
++ - REPEATED_ANT_TEST_METHODS: null
++ - REPEATED_ANT_TEST_COUNT: 500
++ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - CASSANDRA_USE_JDK11: true
++ j8_dtests_vnode:
++ docker:
++ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
++ resource_class: medium
++ working_directory: ~/
++ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
- name: Determine Tests to Run (j8_without_vnodes)
++ name: Determine Tests to Run (j8_with_vnodes)
+ no_output_timeout: 5m
- command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-i [...]
++ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip- [...]
+ - run:
- name: Run dtests (j8_without_vnodes)
++ name: Run dtests (j8_with_vnodes)
+ no_output_timeout: 15m
- command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not [...]
++ command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the e [...]
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
- destination: dtest_j8_without_vnodes
++ destination: dtest_j8_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
- destination: dtest_j8_without_vnodes_logs
++ destination: dtest_j8_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j11_cqlsh-dtests-py2-no-vnodes:
++ j11_cqlshlib_tests:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
- parallelism: 4
++ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
- name: Clone Cassandra dtest Repository (via git)
- command: |
- git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- - run:
- name: Configure virtualenv and python Dependencies
++ name: Run cqlshlib Unit Tests
+ command: |
- # note, this should be super quick as all dependencies should be pre-installed in the docker image
- # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
- # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
- # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
- source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
- pip3 uninstall -y cqlsh
- pip3 freeze
++ time mv ~/cassandra /tmp
++ cd /tmp/cassandra/pylib
++ ./cassandra-cqlsh-tests.sh ..
++ no_output_timeout: 15m
++ - store_test_results:
++ path: /tmp/cassandra/pylib
++ environment:
++ - ANT_HOME: /usr/share/ant
++ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ - LANG: en_US.UTF-8
++ - KEEP_TEST_DIR: true
++ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
++ - PYTHONIOENCODING: utf-8
++ - PYTHONUNBUFFERED: true
++ - CASS_DRIVER_NO_EXTENSIONS: true
++ - CASS_DRIVER_NO_CYTHON: true
++ - CASSANDRA_SKIP_SYNC: true
++ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
++ - DTEST_BRANCH: trunk
++ - CCM_MAX_HEAP_SIZE: 1024M
++ - CCM_HEAP_NEWSIZE: 256M
++ - REPEATED_TESTS_STOP_ON_FAILURE: false
++ - REPEATED_UTESTS: null
++ - REPEATED_UTESTS_COUNT: 500
++ - REPEATED_UTESTS_FQLTOOL: null
++ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
++ - REPEATED_UTESTS_LONG: null
++ - REPEATED_UTESTS_LONG_COUNT: 100
++ - REPEATED_UTESTS_STRESS: null
++ - REPEATED_UTESTS_STRESS_COUNT: 500
++ - REPEATED_JVM_DTESTS: null
++ - REPEATED_JVM_DTESTS_COUNT: 500
++ - REPEATED_JVM_UPGRADE_DTESTS: null
++ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
++ - REPEATED_DTESTS: null
++ - REPEATED_DTESTS_COUNT: 500
++ - REPEATED_UPGRADE_DTESTS: null
++ - REPEATED_UPGRADE_DTESTS_COUNT: 25
++ - REPEATED_ANT_TEST_TARGET: testsome
++ - REPEATED_ANT_TEST_CLASS: null
++ - REPEATED_ANT_TEST_METHODS: null
++ - REPEATED_ANT_TEST_COUNT: 500
++ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - CASSANDRA_USE_JDK11: true
++ j8_jvm_dtests_repeat:
++ docker:
++ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
++ resource_class: medium
++ working_directory: ~/
++ shell: /bin/bash -eo pipefail -l
++ parallelism: 4
++ steps:
++ - attach_workspace:
++ at: /home/cassandra
+ - run:
- name: Determine Tests to Run (j11_without_vnodes)
- no_output_timeout: 5m
- command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource- [...]
++ name: Log Environment Information
++ command: |
++ echo '*** id ***'
++ id
++ echo '*** cat /proc/cpuinfo ***'
++ cat /proc/cpuinfo
++ echo '*** free -m ***'
++ free -m
++ echo '*** df -m ***'
++ df -m
++ echo '*** ifconfig -a ***'
++ ifconfig -a
++ echo '*** uname -a ***'
++ uname -a
++ echo '*** mount ***'
++ mount
++ echo '*** env ***'
++ env
++ echo '*** java ***'
++ which java
++ java -version
+ - run:
- name: Run dtests (j11_without_vnodes)
++ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: |
- echo "cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt"
- cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt
-
- source ~/env3.6/bin/activate
++ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
- if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then
- export CQLSH_PYTHON=/usr/bin/python2.7
++ time mv ~/cassandra /tmp
++ cd /tmp/cassandra
++ if [ -d ~/dtest_jars ]; then
++ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
- java -version
- cd ~/cassandra-dtest
- mkdir -p /tmp/dtest
- java -version
- cd ~/cassandra-dtest
- mkdir -p /tmp/dtest
++ # Calculate the number of test iterations to be run by the current parallel runner.
++ count=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
++ if (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
++ count=$((count+1))
++ fi
- echo "env: $(env)"
- echo "** done env"
- mkdir -p /tmp/results/dtests
- # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
- export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt`
- set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
- echo "env: $(env)"
- echo "** done env"
- mkdir -p /tmp/results/dtests
++ # Put manually specified tests and automatically detected tests together, removing duplicates
++ tests=$(echo ${REPEATED_JVM_DTESTS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
++ echo "Tests to be repeated: ${tests}"
+
- tests_arg=$(echo ${REPEATED_DTESTS} | sed -e "s/,/ /g")
++ # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
++ target=test-jvm-dtest-some
++ testtag=""
++ if [[ $target == "test-cdc" ]]; then
++ testtag="cdc"
++ elif [[ $target == "test-compression" ]]; then
++ testtag="compression"
++ elif [[ $target == "test-system-keyspace-directory" ]]; then
++ testtag="system_keyspace_directory"
++ fi
+
- stop_on_failure_arg=""
- if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
- stop_on_failure_arg="-x"
++ # Run each test class as many times as requested.
++ exit_code="$?"
++ for test in $tests; do
++
++ # Split class and method names from the test name
++ if [[ $test =~ "#" ]]; then
++ class=${test%"#"*}
++ method=${test#*"#"}
++ else
++ class=$test
++ method=""
+ fi
+
- vnodes_args=""
- if true; then
- vnodes_args="--use-vnodes --num-tokens=16"
++ # Prepare the -Dtest.name argument.
++ # It can be the fully qualified class name or the short class name, depending on the target.
++ if [[ $target == "test" || \
++ $target == "test-cdc" || \
++ $target == "test-compression" || \
++ $target == "test-system-keyspace-directory" || \
++ $target == "fqltool-test" || \
++ $target == "long-test" || \
++ $target == "stress-test" ]]; then
++ name_arg="-Dtest.name=${class##*.}"
++ else
++ name_arg="-Dtest.name=$class"
+ fi
+
- upgrade_arg=""
- if false; then
- upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
++ # Prepare the -Dtest.methods argument, which is optional
++ if [[ $method == "" ]]; then
++ methods_arg=""
++ else
++ methods_arg="-Dtest.methods=$method"
+ fi
+
- # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
- set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
- fi
- fi
++ for i in $(seq -w 1 $count); do
++ echo "Running test $test, iteration $i of $count"
++
++ # run the test
++ status="passes"
++ if !( set -o pipefail && \
++ ant test-jvm-dtest-some $name_arg $methods_arg -Dno-build-test=true | \
++ tee stdout.txt \
++ ); then
++ status="fails"
++ exit_code=1
++ fi
++
++ # move the stdout output file
++ dest=/tmp/results/repeated_utests/stdout/${status}/${i}
++ mkdir -p $dest
++ mv stdout.txt $dest/${test}.txt
++
++ # move the XML output files
++ source=build/test/output/${testtag}
++ dest=/tmp/results/repeated_utests/output/${status}/${i}
++ mkdir -p $dest
++ if [[ -d $source && -n "$(ls $source)" ]]; then
++ mv $source/* $dest/
++ fi
++
++ # move the log files
++ source=build/test/logs/${testtag}
++ dest=/tmp/results/repeated_utests/logs/${status}/${i}
++ mkdir -p $dest
++ if [[ -d $source && -n "$(ls $source)" ]]; then
++ mv $source/* $dest/
++ fi
++
++ # maybe stop iterations on test failure
++ if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
++ break
++ fi
++ done
++ done
++ (exit ${exit_code})
- store_test_results:
-- path: /tmp/results
++ path: /tmp/results/repeated_utests/output
- store_artifacts:
-- path: /tmp/dtest
- destination: dtest_j11_without_vnodes
- destination: dtest
++ path: /tmp/results/repeated_utests/stdout
++ destination: stdout
- store_artifacts:
-- path: ~/cassandra-dtest/logs
- destination: dtest_j11_without_vnodes_logs
- destination: dtest_logs
++ path: /tmp/results/repeated_utests/output
++ destination: junitxml
++ - store_artifacts:
++ path: /tmp/results/repeated_utests/logs
++ destination: logs
environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@@ -4092,12 -1209,11 +6246,11 @@@
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
- REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - CASSANDRA_USE_JDK11: true
- j8_dtests_vnode:
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_upgrade_dtests:
++ j11_dtests:
docker:
-- - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
++ - image: apache/cassandra-testing-ubuntu2004-java11:latest
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
@@@ -4105,6 -1221,6 +6258,28 @@@
steps:
- attach_workspace:
at: /home/cassandra
++ - run:
++ name: Log Environment Information
++ command: |
++ echo '*** id ***'
++ id
++ echo '*** cat /proc/cpuinfo ***'
++ cat /proc/cpuinfo
++ echo '*** free -m ***'
++ free -m
++ echo '*** df -m ***'
++ df -m
++ echo '*** ifconfig -a ***'
++ ifconfig -a
++ echo '*** uname -a ***'
++ uname -a
++ echo '*** mount ***'
++ mount
++ echo '*** env ***'
++ env
++ echo '*** java ***'
++ which java
++ java -version
- run:
name: Clone Cassandra dtest Repository (via git)
command: |
@@@ -4118,88 -1234,68 +6293,29 @@@
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
pip3 freeze
- run:
- name: Determine Tests to Run (j8_with_vnodes)
- name: Determine Tests to Run (j8_upgradetests_without_vnodes)
++ name: Determine Tests to Run (j11_without_vnodes)
no_output_timeout: 5m
- command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip- [...]
- command: |
- # reminder: this code (along with all the steps) is independently executed on every circle container
- # so the goal here is to get the circleci script to return the tests *this* container will run
- # which we do via the `circleci` cli tool.
-
- cd cassandra-dtest
- source ~/env3.6/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
-
- if [ -n 'RUN_STATIC_UPGRADE_MATRIX=true' ]; then
- export RUN_STATIC_UPGRADE_MATRIX=true
- fi
-
- echo "***Collected DTests (j8_upgradetests_without_vnodes)***"
- set -eo pipefail && ./run_dtests.py --execute-upgrade-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw --cassandra-dir=../cassandra
- if [ -z '^upgrade_tests' ]; then
- mv /tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw /tmp/all_dtest_tests_j8_upgradetests_without_vnodes
- else
- grep -e '^upgrade_tests' /tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw > /tmp/all_dtest_tests_j8_upgradetests_without_vnodes || { echo "Filter did not match any tests! Exiting build."; exit 0; }
- fi
- set -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_upgradetests_without_vnodes > /tmp/split_dtest_tests_j8_upgradetests_without_vnodes.txt
- cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes.txt | tr '\n' ' ' > /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
- cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
++ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource- [...]
- run:
- name: Run dtests (j8_with_vnodes)
- name: Run dtests (j8_upgradetests_without_vnodes)
++ name: Run dtests (j11_without_vnodes)
no_output_timeout: 15m
- command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the e [...]
- command: |
- echo "cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt"
- cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
-
- source ~/env3.6/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
- if [ -n 'RUN_STATIC_UPGRADE_MATRIX=true' ]; then
- export RUN_STATIC_UPGRADE_MATRIX=true
- fi
-
- java -version
- cd ~/cassandra-dtest
- mkdir -p /tmp/dtest
-
- echo "env: $(env)"
- echo "** done env"
- mkdir -p /tmp/results/dtests
- # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
- export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt`
- set -o pipefail && cd ~/cassandra-dtest && pytest --execute-upgrade-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_upgradetests_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
++ command: "echo \"cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and n [...]
- store_test_results:
path: /tmp/results
- store_artifacts:
path: /tmp/dtest
- destination: dtest_j8_with_vnodes
- destination: dtest_j8_upgradetests_without_vnodes
++ destination: dtest_j11_without_vnodes
- store_artifacts:
path: ~/cassandra-dtest/logs
- destination: dtest_j8_with_vnodes_logs
- environment:
- - ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - LANG: en_US.UTF-8
- - KEEP_TEST_DIR: true
- - DEFAULT_DIR: /home/cassandra/cassandra-dtest
- - PYTHONIOENCODING: utf-8
- - PYTHONUNBUFFERED: true
- - CASS_DRIVER_NO_EXTENSIONS: true
- - CASS_DRIVER_NO_CYTHON: true
- - CASSANDRA_SKIP_SYNC: true
- - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: trunk
- - CCM_MAX_HEAP_SIZE: 1024M
- - CCM_HEAP_NEWSIZE: 256M
- - REPEATED_TESTS_STOP_ON_FAILURE: false
- - REPEATED_UTESTS: null
- - REPEATED_UTESTS_COUNT: 500
- - REPEATED_UTESTS_FQLTOOL: null
- - REPEATED_UTESTS_FQLTOOL_COUNT: 500
- - REPEATED_UTESTS_LONG: null
- - REPEATED_UTESTS_LONG_COUNT: 100
- - REPEATED_UTESTS_STRESS: null
- - REPEATED_UTESTS_STRESS_COUNT: 500
- - REPEATED_JVM_DTESTS: null
- - REPEATED_JVM_DTESTS_COUNT: 500
- - REPEATED_JVM_UPGRADE_DTESTS: null
- - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- - REPEATED_DTESTS: null
- - REPEATED_DTESTS_COUNT: 500
- - REPEATED_UPGRADE_DTESTS: null
- - REPEATED_UPGRADE_DTESTS_COUNT: 25
- - REPEATED_ANT_TEST_TARGET: testsome
- - REPEATED_ANT_TEST_CLASS: null
- - REPEATED_ANT_TEST_METHODS: null
- - REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j11_cqlshlib_tests:
- docker:
- - image: apache/cassandra-testing-ubuntu2004-java11:latest
- resource_class: medium
- working_directory: ~/
- shell: /bin/bash -eo pipefail -l
- parallelism: 1
- steps:
- - attach_workspace:
- at: /home/cassandra
- - run:
- name: Run cqlshlib Unit Tests
- command: |
- export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra/pylib
- ./cassandra-cqlsh-tests.sh ..
- no_output_timeout: 15m
- - store_test_results:
- path: /tmp/cassandra/pylib
- destination: dtest_j8_upgradetests_without_vnodes_logs
++ destination: dtest_j11_without_vnodes_logs
environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@@ -4233,10 -1327,9 +6349,10 @@@
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
- REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_jvm_upgrade_dtests_repeat:
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
- j8_jvm_dtests_repeat:
++ j8_repeated_ant_test:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: medium
@@@ -4269,90 -1362,97 +6385,92 @@@
which java
java -version
- run:
-- name: Repeatedly run new or modifed JUnit tests
++ name: Run repeated JUnit test
no_output_timeout: 15m
command: |
-- set -x
-- export PATH=$JAVA_HOME/bin:$PATH
-- time mv ~/cassandra /tmp
-- cd /tmp/cassandra
-- if [ -d ~/dtest_jars ]; then
-- cp ~/dtest_jars/dtest* /tmp/cassandra/build/
-- fi
--
-- # Calculate the number of test iterations to be run by the current parallel runner.
- count=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
- count=$((${REPEATED_JVM_UPGRADE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_UPGRADE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
-- count=$((count+1))
-- fi
-
- # Put manually specified tests and automatically detected tests together, removing duplicates
- tests=$(echo ${REPEATED_JVM_UPGRADE_DTESTS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
- echo "Tests to be repeated: ${tests}"
++ if [ "${REPEATED_ANT_TEST_CLASS}" == "<nil>" ]; then
++ echo "Repeated utest class name hasn't been defined, exiting without running any test"
++ elif [ "${REPEATED_ANT_TEST_COUNT}" == "<nil>" ]; then
++ echo "Repeated utest count hasn't been defined, exiting without running any test"
++ elif [ "${REPEATED_ANT_TEST_COUNT}" -le 0 ]; then
++ echo "Repeated utest count is lesser or equals than zero, exiting without running any test"
++ else
- # Put manually specified tests and automatically detected tests together, removing duplicates
- tests=$(echo ${REPEATED_JVM_DTESTS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
- echo "Tests to be repeated: ${tests}"
- # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
- target=test-jvm-dtest-some
- testtag=""
- if [[ $target == "test-cdc" ]]; then
- testtag="cdc"
- elif [[ $target == "test-compression" ]]; then
- testtag="compression"
- fi
++ # Calculate the number of test iterations to be run by the current parallel runner.
++ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
++ count=$((${REPEATED_ANT_TEST_COUNT} / CIRCLE_NODE_TOTAL))
++ if (($CIRCLE_NODE_INDEX < (${REPEATED_ANT_TEST_COUNT} % CIRCLE_NODE_TOTAL))); then
++ count=$((count+1))
++ fi
-- # Run each test class as many times as requested.
-- exit_code="$?"
-- for test in $tests; do
++ if (($count <= 0)); then
++ echo "No tests to run in this runner"
++ else
++ echo "Running ${REPEATED_ANT_TEST_TARGET} ${REPEATED_ANT_TEST_CLASS} ${REPEATED_ANT_TEST_METHODS} ${REPEATED_ANT_TEST_COUNT} times"
-- # Split class and method names from the test name
-- if [[ $test =~ "#" ]]; then
-- class=${test%"#"*}
-- method=${test#*"#"}
-- else
-- class=$test
-- method=""
++ set -x
++ export PATH=$JAVA_HOME/bin:$PATH
++ time mv ~/cassandra /tmp
++ cd /tmp/cassandra
++ if [ -d ~/dtest_jars ]; then
++ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
fi
++ target=${REPEATED_ANT_TEST_TARGET}
++ class_path=${REPEATED_ANT_TEST_CLASS}
++ class_name="${class_path##*.}"
++
# Prepare the -Dtest.name argument.
# It can be the fully qualified class name or the short class name, depending on the target.
if [[ $target == "test" || \
$target == "test-cdc" || \
$target == "test-compression" || \
+ $target == "test-system-keyspace-directory" || \
+ $target == "fqltool-test" || \
$target == "long-test" || \
$target == "stress-test" ]]; then
-- name_arg="-Dtest.name=${class##*.}"
++ name="-Dtest.name=$class_name"
else
-- name_arg="-Dtest.name=$class"
++ name="-Dtest.name=$class_path"
fi
# Prepare the -Dtest.methods argument, which is optional
- if [ $method == "" ]; then
- if [[ $method == "" ]]; then
-- methods_arg=""
++ if [ "${REPEATED_ANT_TEST_METHODS}" == "<nil>" ]; then
++ methods=""
else
-- methods_arg="-Dtest.methods=$method"
++ methods="-Dtest.methods=${REPEATED_ANT_TEST_METHODS}"
fi
++ # Run the test target as many times as requested collecting the exit code,
++ # stopping the iteration only if stop_on_failure is set.
++ exit_code="$?"
for i in $(seq -w 1 $count); do
-- echo "Running test $test, iteration $i of $count"
++
++ echo "Running test iteration $i of $count"
# run the test
status="passes"
-- if !( set -o pipefail && \
-- ant test-jvm-dtest-some $name_arg $methods_arg -Dno-build-test=true | \
-- tee stdout.txt \
-- ); then
++ if !( set -o pipefail && ant $target $name $methods -Dno-build-test=true | tee stdout.txt ); then
status="fails"
exit_code=1
fi
# move the stdout output file
-- dest=/tmp/results/repeated_utests/stdout/${status}/${i}
++ dest=/tmp/results/repeated_utest/stdout/${status}/${i}
mkdir -p $dest
-- mv stdout.txt $dest/${test}.txt
++ mv stdout.txt $dest/${REPEATED_ANT_TEST_TARGET}-${REPEATED_ANT_TEST_CLASS}.txt
# move the XML output files
- source=build/test/output/${testtag}
- dest=/tmp/results/repeated_utests/output/${status}/${i}
+ source=build/test/output
- dest=/tmp/results/repeated_utests/output/${status}/${i}
++ dest=/tmp/results/repeated_utest/output/${status}/${i}
mkdir -p $dest
if [[ -d $source && -n "$(ls $source)" ]]; then
mv $source/* $dest/
fi
# move the log files
- source=build/test/logs/${testtag}
- dest=/tmp/results/repeated_utests/logs/${status}/${i}
+ source=build/test/logs
- dest=/tmp/results/repeated_utests/logs/${status}/${i}
++ dest=/tmp/results/repeated_utest/logs/${status}/${i}
mkdir -p $dest
if [[ -d $source && -n "$(ls $source)" ]]; then
mv $source/* $dest/
@@@ -4363,126 -1463,22 +6481,93 @@@
break
fi
done
-- done
-- (exit ${exit_code})
++
++ (exit ${exit_code})
++ fi
++ fi
- store_test_results:
-- path: /tmp/results/repeated_utests/output
++ path: /tmp/results/repeated_utest/output
- store_artifacts:
-- path: /tmp/results/repeated_utests/stdout
++ path: /tmp/results/repeated_utest/stdout
destination: stdout
- store_artifacts:
-- path: /tmp/results/repeated_utests/output
++ path: /tmp/results/repeated_utest/output
destination: junitxml
- store_artifacts:
-- path: /tmp/results/repeated_utests/logs
++ path: /tmp/results/repeated_utest/logs
destination: logs
environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j11_dtests:
++ j8_utests_long:
+ docker:
- - image: apache/cassandra-testing-ubuntu2004-java11:latest
++ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
- parallelism: 4
++ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
- name: Log Environment Information
- command: |
- echo '*** id ***'
- id
- echo '*** cat /proc/cpuinfo ***'
- cat /proc/cpuinfo
- echo '*** free -m ***'
- free -m
- echo '*** df -m ***'
- df -m
- echo '*** ifconfig -a ***'
- ifconfig -a
- echo '*** uname -a ***'
- uname -a
- echo '*** mount ***'
- mount
- echo '*** env ***'
- env
- echo '*** java ***'
- which java
- java -version
- - run:
- name: Clone Cassandra dtest Repository (via git)
- command: |
- git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- - run:
- name: Configure virtualenv and python Dependencies
++ name: Run Unit Tests (long-test)
+ command: |
- # note, this should be super quick as all dependencies should be pre-installed in the docker image
- # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
- # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
- # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
- source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
- pip3 uninstall -y cqlsh
- pip3 freeze
- - run:
- name: Determine Tests to Run (j11_without_vnodes)
- no_output_timeout: 5m
- command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource- [...]
- - run:
- name: Run dtests (j11_without_vnodes)
++ time mv ~/cassandra /tmp
++ cd /tmp/cassandra
++ if [ -d ~/dtest_jars ]; then
++ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
++ fi
++ ant long-test
+ no_output_timeout: 15m
- command: "echo \"cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and n [...]
+ - store_test_results:
- path: /tmp/results
++ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
- path: /tmp/dtest
- destination: dtest_j11_without_vnodes
++ path: /tmp/cassandra/build/test/output
++ destination: junitxml
+ - store_artifacts:
- path: ~/cassandra-dtest/logs
- destination: dtest_j11_without_vnodes_logs
++ path: /tmp/cassandra/build/test/logs
++ destination: logs
+ environment:
- ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@@ -4516,10 -1510,9 +6601,9 @@@
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
- REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - CASSANDRA_USE_JDK11: true
- j8_repeated_ant_test:
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- utests_cdc:
++ j8_utests_cdc:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: medium
@@@ -4552,121 -1564,32 +6655,33 @@@
which java
java -version
- run:
- name: Run repeated JUnit test
- no_output_timeout: 15m
+ name: Run Unit Tests (testclasslist-cdc)
command: |
- if [ "${REPEATED_ANT_TEST_CLASS}" == "<nil>" ]; then
- echo "Repeated utest class name hasn't been defined, exiting without running any test"
- elif [ "${REPEATED_ANT_TEST_COUNT}" == "<nil>" ]; then
- echo "Repeated utest count hasn't been defined, exiting without running any test"
- elif [ "${REPEATED_ANT_TEST_COUNT}" -le 0 ]; then
- echo "Repeated utest count is lesser or equals than zero, exiting without running any test"
- else
-
- # Calculate the number of test iterations to be run by the current parallel runner.
- # Since we are running the same test multiple times there is no need to use `circleci tests split`.
- count=$((${REPEATED_ANT_TEST_COUNT} / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (${REPEATED_ANT_TEST_COUNT} % CIRCLE_NODE_TOTAL))); then
- count=$((count+1))
- fi
-
- if (($count <= 0)); then
- echo "No tests to run in this runner"
- else
- echo "Running ${REPEATED_ANT_TEST_TARGET} ${REPEATED_ANT_TEST_CLASS} ${REPEATED_ANT_TEST_METHODS} ${REPEATED_ANT_TEST_COUNT} times"
-
- set -x
- export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- if [ -d ~/dtest_jars ]; then
- cp ~/dtest_jars/dtest* /tmp/cassandra/build/
- fi
-
- target=${REPEATED_ANT_TEST_TARGET}
- class_path=${REPEATED_ANT_TEST_CLASS}
- class_name="${class_path##*.}"
-
- # Prepare the -Dtest.name argument.
- # It can be the fully qualified class name or the short class name, depending on the target.
- if [[ $target == "test" || \
- $target == "test-cdc" || \
- $target == "test-compression" || \
- $target == "test-system-keyspace-directory" || \
- $target == "fqltool-test" || \
- $target == "long-test" || \
- $target == "stress-test" ]]; then
- name="-Dtest.name=$class_name"
- else
- name="-Dtest.name=$class_path"
- fi
-
- # Prepare the -Dtest.methods argument, which is optional
- if [ "${REPEATED_ANT_TEST_METHODS}" == "<nil>" ]; then
- methods=""
- else
- methods="-Dtest.methods=${REPEATED_ANT_TEST_METHODS}"
- fi
-
- # Run the test target as many times as requested collecting the exit code,
- # stopping the iteration only if stop_on_failure is set.
- exit_code="$?"
- for i in $(seq -w 1 $count); do
-
- echo "Running test iteration $i of $count"
-
- # run the test
- status="passes"
- if !( set -o pipefail && ant $target $name $methods -Dno-build-test=true | tee stdout.txt ); then
- status="fails"
- exit_code=1
- fi
-
- # move the stdout output file
- dest=/tmp/results/repeated_utest/stdout/${status}/${i}
- mkdir -p $dest
- mv stdout.txt $dest/${REPEATED_ANT_TEST_TARGET}-${REPEATED_ANT_TEST_CLASS}.txt
-
- # move the XML output files
- source=build/test/output
- dest=/tmp/results/repeated_utest/output/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # move the log files
- source=build/test/logs
- dest=/tmp/results/repeated_utest/logs/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # maybe stop iterations on test failure
- if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
- break
- fi
- done
-
- (exit ${exit_code})
- fi
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
fi
+ ant testclasslist-cdc -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
+ no_output_timeout: 15m
- store_test_results:
- path: /tmp/results/repeated_utest/output
- - store_artifacts:
- path: /tmp/results/repeated_utest/stdout
- destination: stdout
+ path: /tmp/cassandra/build/test/output/
- store_artifacts:
- path: /tmp/results/repeated_utest/output
+ path: /tmp/cassandra/build/test/output
destination: junitxml
- store_artifacts:
- path: /tmp/results/repeated_utest/logs
+ path: /tmp/cassandra/build/test/logs
destination: logs
environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@@ -5081,106 -1908,18 +7096,38 @@@
- run:
name: Run dtests (j8_with_vnodes)
no_output_timeout: 15m
- command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the e [...]
+ command: |
+ echo "cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt
+
- source ~/env3.8/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
- if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then
- export CQLSH_PYTHON=/usr/bin/python3.8
- fi
-
- java -version
- cd ~/cassandra-dtest
- mkdir -p /tmp/dtest
-
- echo "env: $(env)"
- echo "** done env"
- mkdir -p /tmp/results/dtests
- # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
- export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`
- set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
- - store_test_results:
- path: /tmp/results
- - store_artifacts:
- path: /tmp/dtest
- destination: dtest_j8_with_vnodes
- - store_artifacts:
- path: ~/cassandra-dtest/logs
- destination: dtest_j8_with_vnodes_logs
- environment:
- - ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - LANG: en_US.UTF-8
- - KEEP_TEST_DIR: true
- - DEFAULT_DIR: /home/cassandra/cassandra-dtest
- - PYTHONIOENCODING: utf-8
- - PYTHONUNBUFFERED: true
- - CASS_DRIVER_NO_EXTENSIONS: true
- - CASS_DRIVER_NO_CYTHON: true
- - CASSANDRA_SKIP_SYNC: true
- - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: trunk
- - CCM_MAX_HEAP_SIZE: 1024M
- - CCM_HEAP_NEWSIZE: 256M
- - REPEATED_TESTS_STOP_ON_FAILURE: false
- - REPEATED_UTESTS: null
- - REPEATED_UTESTS_COUNT: 500
- - REPEATED_UTESTS_FQLTOOL: null
- - REPEATED_UTESTS_FQLTOOL_COUNT: 500
- - REPEATED_UTESTS_LONG: null
- - REPEATED_UTESTS_LONG_COUNT: 100
- - REPEATED_UTESTS_STRESS: null
- - REPEATED_UTESTS_STRESS_COUNT: 500
- - REPEATED_JVM_DTESTS: null
- - REPEATED_JVM_DTESTS_COUNT: 500
- - REPEATED_JVM_UPGRADE_DTESTS: null
- - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- - REPEATED_DTESTS: null
- - REPEATED_DTESTS_COUNT: 500
- - REPEATED_UPGRADE_DTESTS: null
- - REPEATED_UPGRADE_DTESTS_COUNT: 25
- - REPEATED_ANT_TEST_TARGET: testsome
- - REPEATED_ANT_TEST_CLASS: null
- - REPEATED_ANT_TEST_METHODS: null
- - REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- utests_long:
- docker:
- - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
- resource_class: medium
- working_directory: ~/
- shell: /bin/bash -eo pipefail -l
- parallelism: 1
- steps:
- - attach_workspace:
- at: /home/cassandra
- - run:
- name: Run Unit Tests (long-test)
- command: |
- export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- if [ -d ~/dtest_jars ]; then
- cp ~/dtest_jars/dtest* /tmp/cassandra/build/
++ source ~/env3.8/bin/activate
++ export PATH=$JAVA_HOME/bin:$PATH
++ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then
++ export CQLSH_PYTHON=/usr/bin/python3.8
+ fi
- ant long-test
- no_output_timeout: 15m
++
++ java -version
++ cd ~/cassandra-dtest
++ mkdir -p /tmp/dtest
++
++ echo "env: $(env)"
++ echo "** done env"
++ mkdir -p /tmp/results/dtests
++ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
++ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`
++ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
- store_test_results:
- path: /tmp/cassandra/build/test/output/
+ path: /tmp/results
- store_artifacts:
- path: /tmp/cassandra/build/test/output
- destination: junitxml
+ path: /tmp/dtest
+ destination: dtest_j8_with_vnodes
- store_artifacts:
- path: /tmp/cassandra/build/test/logs
- destination: logs
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j8_with_vnodes_logs
environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@@ -5267,9 -2004,18 +7214,20 @@@
fi
# Put manually specified tests and automatically detected tests together, removing duplicates
- tests=$(echo ${REPEATED_JVM_DTESTS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
+ tests=$(echo ${REPEATED_UTESTS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
echo "Tests to be repeated: ${tests}"
+ # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
- target=test-jvm-dtest-some
++ target=testsome
+ testtag=""
+ if [[ $target == "test-cdc" ]]; then
+ testtag="cdc"
+ elif [[ $target == "test-compression" ]]; then
+ testtag="compression"
++ elif [[ $target == "test-system-keyspace-directory" ]]; then
++ testtag="system_keyspace_directory"
+ fi
+
# Run each test class as many times as requested.
exit_code="$?"
for test in $tests; do
@@@ -5393,88 -2134,153 +7351,42 @@@
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
- REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_repeated_ant_test:
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
- utests_system_keyspace_directory:
++ j8_utests_fqltool:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
-- parallelism: 4
++ parallelism: 1
steps:
- attach_workspace:
at: /home/cassandra
- run:
- name: Determine unit Tests to Run
- command: |
- # reminder: this code (along with all the steps) is independently executed on every circle container
- # so the goal here is to get the circleci script to return the tests *this* container will run
- # which we do via the `circleci` cli tool.
-
- rm -fr ~/cassandra-dtest/upgrade_tests
- echo "***java tests***"
-
- # get all of our unit test filenames
- set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
-
- # split up the unit tests into groups based on the number of containers we have
- set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
- set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
- echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
- cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
- no_output_timeout: 15m
- - run:
-- name: Log Environment Information
-- command: |
-- echo '*** id ***'
-- id
-- echo '*** cat /proc/cpuinfo ***'
-- cat /proc/cpuinfo
-- echo '*** free -m ***'
-- free -m
-- echo '*** df -m ***'
-- df -m
-- echo '*** ifconfig -a ***'
-- ifconfig -a
-- echo '*** uname -a ***'
-- uname -a
-- echo '*** mount ***'
-- mount
-- echo '*** env ***'
-- env
-- echo '*** java ***'
-- which java
-- java -version
-- - run:
- name: Run Unit Tests (testclasslist-system-keyspace-directory)
- name: Run repeated JUnit test
- no_output_timeout: 15m
++ name: Run Unit Tests (fqltool-test)
command: |
- set -x
- if [ "${REPEATED_ANT_TEST_CLASS}" == "<nil>" ]; then
- echo "Repeated utest class name hasn't been defined, exiting without running any test"
- elif [ "${REPEATED_ANT_TEST_COUNT}" == "<nil>" ]; then
- echo "Repeated utest count hasn't been defined, exiting without running any test"
- elif [ "${REPEATED_ANT_TEST_COUNT}" -le 0 ]; then
- echo "Repeated utest count is lesser or equals than zero, exiting without running any test"
- else
-
- # Calculate the number of test iterations to be run by the current parallel runner.
- # Since we are running the same test multiple times there is no need to use `circleci tests split`.
- count=$((${REPEATED_ANT_TEST_COUNT} / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (${REPEATED_ANT_TEST_COUNT} % CIRCLE_NODE_TOTAL))); then
- count=$((count+1))
- fi
-
- if (($count <= 0)); then
- echo "No tests to run in this runner"
- else
- echo "Running ${REPEATED_ANT_TEST_TARGET} ${REPEATED_ANT_TEST_CLASS} ${REPEATED_ANT_TEST_METHODS} ${REPEATED_ANT_TEST_COUNT} times"
-
- set -x
- export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- if [ -d ~/dtest_jars ]; then
- cp ~/dtest_jars/dtest* /tmp/cassandra/build/
- fi
-
- target=${REPEATED_ANT_TEST_TARGET}
- class_path=${REPEATED_ANT_TEST_CLASS}
- class_name="${class_path##*.}"
-
- # Prepare the -Dtest.name argument.
- # It can be the fully qualified class name or the short class name, depending on the target.
- if [[ $target == "test" || \
- $target == "test-cdc" || \
- $target == "test-compression" || \
- $target == "long-test" || \
- $target == "stress-test" ]]; then
- name="-Dtest.name=$class_name"
- else
- name="-Dtest.name=$class_path"
- fi
-
- # Prepare the -Dtest.methods argument, which is optional
- if [ "${REPEATED_ANT_TEST_METHODS}" == "<nil>" ]; then
- methods=""
- else
- methods="-Dtest.methods=${REPEATED_ANT_TEST_METHODS}"
- fi
-
- # Run the test target as many times as requested collecting the exit code,
- # stopping the iteration only if stop_on_failure is set.
- exit_code="$?"
- for i in $(seq -w 1 $count); do
-
- echo "Running test iteration $i of $count"
-
- # run the test
- status="passes"
- if !( set -o pipefail && ant $target $name $methods -Dno-build-test=true | tee stdout.txt ); then
- status="fails"
- exit_code=1
- fi
-
- # move the stdout output file
- dest=/tmp/results/repeated_utest/stdout/${status}/${i}
- mkdir -p $dest
- mv stdout.txt $dest/${REPEATED_ANT_TEST_TARGET}-${REPEATED_ANT_TEST_CLASS}.txt
-
- # move the XML output files
- source=build/test/output
- dest=/tmp/results/repeated_utest/output/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # move the log files
- source=build/test/logs
- dest=/tmp/results/repeated_utest/logs/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # maybe stop iterations on test failure
- if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
- break
- fi
- done
-
- (exit ${exit_code})
- fi
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
fi
- test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
- if [ -z "$test_timeout" ]; then
- test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
- fi
- ant testclasslist-system-keyspace-directory -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
++ ant fqltool-test
+ no_output_timeout: 15m
- store_test_results:
- path: /tmp/results/repeated_utest/output
- - store_artifacts:
- path: /tmp/results/repeated_utest/stdout
- destination: stdout
+ path: /tmp/cassandra/build/test/output/
- store_artifacts:
- path: /tmp/results/repeated_utest/output
+ path: /tmp/cassandra/build/test/output
destination: junitxml
- store_artifacts:
- path: /tmp/results/repeated_utest/logs
+ path: /tmp/cassandra/build/test/logs
destination: logs
environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@@ -5516,92 -2320,92 +7428,127 @@@
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Run cqlshlib Unit Tests
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra/pylib
+ ./cassandra-cqlsh-tests.sh ..
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/pylib
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- utests_fqltool:
++ j11_dtests_vnode:
+ docker:
- - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
++ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
- parallelism: 1
+ parallelism: 4
steps:
- attach_workspace:
at: /home/cassandra
+ - run:
- name: Run Unit Tests (fqltool-test)
++ name: Log Environment Information
++ command: |
++ echo '*** id ***'
++ id
++ echo '*** cat /proc/cpuinfo ***'
++ cat /proc/cpuinfo
++ echo '*** free -m ***'
++ free -m
++ echo '*** df -m ***'
++ df -m
++ echo '*** ifconfig -a ***'
++ ifconfig -a
++ echo '*** uname -a ***'
++ uname -a
++ echo '*** mount ***'
++ mount
++ echo '*** env ***'
++ env
++ echo '*** java ***'
++ which java
++ java -version
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- if [ -d ~/dtest_jars ]; then
- cp ~/dtest_jars/dtest* /tmp/cassandra/build/
- fi
- ant fqltool-test
- pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
++ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
++ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
- name: Run repeated Python dtest
++ name: Determine Tests to Run (j11_with_vnodes)
++ no_output_timeout: 5m
++ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip [...]
++ - run:
++ name: Run dtests (j11_with_vnodes)
no_output_timeout: 15m
- command: |
- if [ "${REPEATED_DTESTS}" == "<nil>" ]; then
- echo "Repeated dtest name hasn't been defined, exiting without running any test"
- elif [ "${REPEATED_DTESTS_COUNT}" == "<nil>" ]; then
- echo "Repeated dtest count hasn't been defined, exiting without running any test"
- elif [ "${REPEATED_DTESTS_COUNT}" -le 0 ]; then
- echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
- else
-
- # Calculate the number of test iterations to be run by the current parallel runner.
- # Since we are running the same test multiple times there is no need to use `circleci tests split`.
- count=$((${REPEATED_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (${REPEATED_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
- count=$((count+1))
- fi
-
- if (($count <= 0)); then
- echo "No tests to run in this runner"
- else
- echo "Running ${REPEATED_DTESTS} $count times"
-
- source ~/env3.6/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
-
- java -version
- cd ~/cassandra-dtest
- mkdir -p /tmp/dtest
-
- echo "env: $(env)"
- echo "** done env"
- mkdir -p /tmp/results/dtests
-
- tests_arg=$(echo ${REPEATED_DTESTS} | sed -e "s/,/ /g")
-
- stop_on_failure_arg=""
- if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
- stop_on_failure_arg="-x"
- fi
-
- vnodes_args=""
- if false; then
- vnodes_args="--use-vnodes --num-tokens=16"
- fi
-
- upgrade_arg=""
- if false; then
- upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
- fi
-
- # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
- set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
- fi
- fi
++ command: "echo \"cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the [...]
- store_test_results:
- path: /tmp/cassandra/build/test/output/
+ path: /tmp/results
- store_artifacts:
- path: /tmp/cassandra/build/test/output
- destination: junitxml
+ path: /tmp/dtest
- destination: dtest
++ destination: dtest_j11_with_vnodes
- store_artifacts:
- path: /tmp/cassandra/build/test/logs
- destination: logs
+ path: ~/cassandra-dtest/logs
- destination: dtest_logs
++ destination: dtest_j11_with_vnodes_logs
environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@@ -5635,11 -2437,11 +7582,12 @@@
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
- REPEATED_ANT_TEST_COUNT: 500
-- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
-- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j11_dtests_vnode:
- utests_cdc_repeat:
++ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - CASSANDRA_USE_JDK11: true
... 29216 lines suppressed ...
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org