You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by ma...@apache.org on 2019/03/15 10:06:20 UTC
[cassandra] branch cassandra-3.11 updated (84fc68c -> 0c2d992)
This is an automated email from the ASF dual-hosted git repository.
marcuse pushed a change to branch cassandra-3.11
in repository https://gitbox.apache.org/repos/asf/cassandra.git.
from 84fc68c Fix copyright years in generated apidocs and in the NOTICE file
new 7333c2b Refactor Circle CI configuration
new 7bda2a0 Merge branch 'cassandra-2.2' into cassandra-3.0
new 0c2d992 Merge branch 'cassandra-3.0' into cassandra-3.11
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.circleci/config-2_1.yml | 463 ++++++++++
.circleci/config-2_1.yml.high_res.patch | 16 +
.circleci/config.yml | 1528 +++++++++++++++++++++++--------
.circleci/config.yml.HIGHRES | 1160 +++++++++++++++++++++++
.circleci/config.yml.LOWRES | 1160 +++++++++++++++++++++++
.circleci/generate.sh | 9 +
.circleci/readme.md | 32 +
CHANGES.txt | 1 +
build.xml | 11 +
9 files changed, 3978 insertions(+), 402 deletions(-)
create mode 100644 .circleci/config-2_1.yml
create mode 100644 .circleci/config-2_1.yml.high_res.patch
create mode 100644 .circleci/config.yml.HIGHRES
create mode 100644 .circleci/config.yml.LOWRES
create mode 100755 .circleci/generate.sh
create mode 100644 .circleci/readme.md
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org
[cassandra] 01/01: Merge branch 'cassandra-3.0' into cassandra-3.11
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
marcuse pushed a commit to branch cassandra-3.11
in repository https://gitbox.apache.org/repos/asf/cassandra.git
commit 0c2d99244d9161b2e0ed15b228755881b6d16b52
Merge: 84fc68c 7bda2a0
Author: Marcus Eriksson <ma...@apache.org>
AuthorDate: Fri Mar 15 09:56:16 2019 +0000
Merge branch 'cassandra-3.0' into cassandra-3.11
.circleci/config-2_1.yml | 463 ++++++++++
.circleci/config-2_1.yml.high_res.patch | 16 +
.circleci/config.yml | 1528 +++++++++++++++++++++++--------
.circleci/config.yml.HIGHRES | 1160 +++++++++++++++++++++++
.circleci/config.yml.LOWRES | 1160 +++++++++++++++++++++++
.circleci/generate.sh | 9 +
.circleci/readme.md | 32 +
CHANGES.txt | 1 +
build.xml | 11 +
9 files changed, 3978 insertions(+), 402 deletions(-)
diff --cc .circleci/config-2_1.yml
index 0000000,859100d..5d8d884
mode 000000,100644..100644
--- a/.circleci/config-2_1.yml
+++ b/.circleci/config-2_1.yml
@@@ -1,0 -1,448 +1,463 @@@
+ version: 2.1
+
+ default_env_vars: &default_env_vars
+ JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ ANT_HOME: /usr/share/ant
+ LANG: en_US.UTF-8
+ KEEP_TEST_DIR: true
+ DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ PYTHONIOENCODING: utf-8
+ PYTHONUNBUFFERED: true
+ CASS_DRIVER_NO_EXTENSIONS: true
+ CASS_DRIVER_NO_CYTHON: true
+ #Skip all syncing to disk to avoid performance issues in flaky CI environments
+ CASSANDRA_SKIP_SYNC: true
+ DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ DTEST_BRANCH: master
+ CCM_MAX_HEAP_SIZE: 1024M
+ CCM_HEAP_NEWSIZE: 256M
+
+ j8_par_executor: &j8_par_executor
+ executor:
+ name: java8-executor
+ #exec_resource_class: xlarge
+ parallelism: 4
+
+ j8_seq_executor: &j8_seq_executor
+ executor:
+ name: java8-executor
+ #exec_resource_class: xlarge
+ parallelism: 1 # sequential, single container tests: no parallelism benefits
+
+ with_dtests_jobs: &with_dtest_jobs
+ jobs:
+ - build
+ # Java 8 unit tests will be run automatically
+ - j8_unit_tests:
+ requires:
+ - build
+ - j8_jvm_dtests:
+ requires:
+ - build
+ # specialized unit tests (all run on request using Java 8)
+ - start_utests_long:
+ type: approval
+ requires:
+ - build
+ - utests_long:
+ requires:
+ - start_utests_long
+ - start_utests_compression:
+ type: approval
+ requires:
+ - build
+ - utests_compression:
+ requires:
+ - start_utests_compression
++ - start_utests_stress:
++ type: approval
++ requires:
++ - build
++ - utests_stress:
++ requires:
++ - start_utests_stress
+ # Java 8 dtests (on request)
+ - start_j8_dtests:
+ type: approval
+ requires:
+ - build
+ - j8_dtests-with-vnodes:
+ requires:
+ - start_j8_dtests
+ - j8_dtests-no-vnodes:
+ requires:
+ - start_j8_dtests
+ # Java 8 upgrade tests
+ - start_upgrade_tests:
+ type: approval
+ requires:
+ - build
+ - j8_upgradetests-no-vnodes:
+ requires:
+ - start_upgrade_tests
+
+ with_dtest_jobs_only: &with_dtest_jobs_only
+ jobs:
+ - build
+ - j8_dtests-with-vnodes:
+ requires:
+ - build
+ - j8_dtests-no-vnodes:
+ requires:
+ - build
+
+ workflows:
+ version: 2
+ build_and_run_tests: *with_dtest_jobs
+ #build_and_run_tests: *with_dtest_jobs_only
+
+ executors:
+ java8-executor:
+ parameters:
+ exec_resource_class:
+ type: string
+ default: medium
+ docker:
+ - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ resource_class: << parameters.exec_resource_class >>
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ environment:
+ <<: *default_env_vars
+ JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+
+ jobs:
+ build:
+ executor: java8-executor
+ parallelism: 1 # This job doesn't benefit from parallelism
+ steps:
+ - log_environment
+ - clone_cassandra
+ - build_cassandra
+ - run_eclipse_warnings
+ - persist_to_workspace:
+ root: /home/cassandra
+ paths:
+ - cassandra
+ - .m2
+
+ j8_unit_tests:
+ <<: *j8_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - create_junit_containers
+ - log_environment
+ - run_parallel_junit_tests
+
+ j8_jvm_dtests:
+ <<: *j8_seq_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run_junit_tests:
+ target: test-jvm-dtest-forking
+
+ utests_long:
+ <<: *j8_seq_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run_junit_tests:
+ target: long-test
+
+ utests_compression:
+ <<: *j8_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - create_junit_containers
+ - log_environment
+ - run_parallel_junit_tests:
+ target: testclasslist-compression
+
++ utests_stress:
++ <<: *j8_seq_executor
++ steps:
++ - attach_workspace:
++ at: /home/cassandra
++ - run_junit_tests:
++ target: stress-test
++
+ j8_dtests-with-vnodes:
+ <<: *j8_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - clone_dtest
+ - create_venv
+ - create_dtest_containers:
+ file_tag: j8_with_vnodes
+ run_dtests_extra_args: '--use-vnodes --skip-resource-intensive-tests'
+ - run_dtests:
+ file_tag: j8_with_vnodes
+ pytest_extra_args: '--use-vnodes --num-tokens=32 --skip-resource-intensive-tests'
+
+ j8_dtests-no-vnodes:
+ <<: *j8_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - clone_dtest
+ - create_venv
+ - create_dtest_containers:
+ file_tag: j8_without_vnodes
+ run_dtests_extra_args: '--skip-resource-intensive-tests'
+ - run_dtests:
+ file_tag: j8_without_vnodes
+ pytest_extra_args: '--skip-resource-intensive-tests'
+
+ j8_upgradetests-no-vnodes:
+ <<: *j8_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - clone_dtest
+ - create_venv
+ - create_dtest_containers:
+ file_tag: j8_upgradetests_without_vnodes
+ run_dtests_extra_args: '--execute-upgrade-tests'
+ extra_env_args: 'RUN_STATIC_UPGRADE_MATRIX=true'
+ tests_filter_pattern: '^upgrade_tests'
+ - run_dtests:
+ file_tag: j8_upgradetests_without_vnodes
+ extra_env_args: 'RUN_STATIC_UPGRADE_MATRIX=true'
+ pytest_extra_args: '--execute-upgrade-tests'
+
+ commands:
+ log_environment:
+ steps:
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+
+ clone_cassandra:
+ steps:
+ - run:
+ name: Clone Cassandra Repository (via git)
+ command: |
+ git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH git://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra
+
+ clone_dtest:
+ steps:
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+
+ build_cassandra:
+ steps:
+ - run:
+ name: Build Cassandra
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ cd ~/cassandra
+ # Loop to prevent failure due to maven-ant-tasks not downloading a jar..
+ for x in $(seq 1 3); do
+ ${ANT_HOME}/bin/ant clean jar
+ RETURN="$?"
+ if [ "${RETURN}" -eq "0" ]; then
+ break
+ fi
+ done
+ # Exit, if we didn't build successfully
+ if [ "${RETURN}" -ne "0" ]; then
+ echo "Build failed with exit code: ${RETURN}"
+ exit ${RETURN}
+ fi
+ no_output_timeout: 15m
+
+ run_eclipse_warnings:
+ steps:
+ - run:
+ name: Run eclipse-warnings
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ cd ~/cassandra
+ ant eclipse-warnings
+
+ create_junit_containers:
+ steps:
+ - run:
+ name: Determine Unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | cut -c 37-1000000 | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+
+ no_output_timeout: 15m
+
+ run_junit_tests:
+ parameters:
+ target:
+ type: string
+ no_output_timeout:
+ type: string
+ default: 15m
+ steps:
+ - run:
+ name: Run Unit Tests (<<parameters.target>>)
+ # Please note that we run `clean` and therefore rebuild the project, as we can't run tests on Java 8 in case
+ # based on Java 11 builds.
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ ant clean <<parameters.target>>
+ no_output_timeout: <<parameters.no_output_timeout>>
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+
+ run_parallel_junit_tests:
+ parameters:
+ target:
+ type: string
+ default: testclasslist
+ no_output_timeout:
+ type: string
+ default: 15m
+ steps:
+ - run:
+ name: Run Unit Tests (<<parameters.target>>)
+ # Please note that we run `clean` and therefore rebuild the project, as we can't run tests on Java 8 in case
+ # based on Java 11 builds.
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ ant <<parameters.target>> -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
+ no_output_timeout: <<parameters.no_output_timeout>>
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+
+ create_venv:
+ steps:
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 freeze
+
+ create_dtest_containers:
+ parameters:
+ file_tag:
+ type: string
+ run_dtests_extra_args:
+ type: string
+ default: ''
+ extra_env_args:
+ type: string
+ default: ''
+ tests_filter_pattern:
+ type: string
+ default: ''
+ steps:
+ - run:
+ name: Determine Tests to Run (<<parameters.file_tag>>)
+ no_output_timeout: 5m
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ cd cassandra-dtest
+ source ~/env/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+
+ if [ -n '<<parameters.extra_env_args>>' ]; then
+ export <<parameters.extra_env_args>>
+ fi
+
+ echo "***Collected DTests (<<parameters.file_tag>>)***"
+ set -eo pipefail && ./run_dtests.py <<parameters.run_dtests_extra_args>> --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_<<parameters.file_tag>>_raw --cassandra-dir=../cassandra
+ if [ -z '<<parameters.tests_filter_pattern>>' ]; then
+ mv /tmp/all_dtest_tests_<<parameters.file_tag>>_raw /tmp/all_dtest_tests_<<parameters.file_tag>>
+ else
+ grep -e '<<parameters.tests_filter_pattern>>' /tmp/all_dtest_tests_<<parameters.file_tag>>_raw > /tmp/all_dtest_tests_<<parameters.file_tag>> || { echo "Filter did not match any tests! Exiting build."; exit 0; }
+ fi
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_<<parameters.file_tag>> > /tmp/split_dtest_tests_<<parameters.file_tag>>.txt
+ cat /tmp/split_dtest_tests_<<parameters.file_tag>>.txt | tr '\n' ' ' > /tmp/split_dtest_tests_<<parameters.file_tag>>_final.txt
+ cat /tmp/split_dtest_tests_<<parameters.file_tag>>_final.txt
+
+ run_dtests:
+ parameters:
+ file_tag:
+ type: string
+ pytest_extra_args:
+ type: string
+ default: ''
+ extra_env_args:
+ type: string
+ default: ''
+ steps:
+ - run:
+ name: Run dtests (<<parameters.file_tag>>)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_<<parameters.file_tag>>_final.txt"
+ cat /tmp/split_dtest_tests_<<parameters.file_tag>>_final.txt
+
+ source ~/env/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n '<<parameters.extra_env_args>>' ]; then
+ export <<parameters.extra_env_args>>
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_<<parameters.file_tag>>_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest <<parameters.pytest_extra_args>> --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_<<parameters.file_tag>>.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_<<parameters.file_tag>>
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_<<parameters.file_tag>>_logs
diff --cc .circleci/config.yml
index 89c25cd,0d56e23..7016d9b
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@@ -1,436 -1,1095 +1,1160 @@@
version: 2
jobs:
build:
- <<: *env_settings
- parallelism: 1 # This job doesn't benefit from parallelism
+ docker:
+ - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
- docker:
- - image: *docker_image
- environment:
- <<: *env_vars
+ parallelism: 1
steps:
- - run:
- name: Log Environment Information
- command: |
- echo '*** id ***'
- id
- echo '*** cat /proc/cpuinfo ***'
- cat /proc/cpuinfo
- echo '*** free -m ***'
- free -m
- echo '*** df -m ***'
- df -m
- echo '*** ifconfig -a ***'
- ifconfig -a
- echo '*** uname -a ***'
- uname -a
- echo '*** mount ***'
- mount
- echo '*** env ***'
- env
- - run:
- name: Clone Cassandra Repository (via git)
- command: |
- export LANG=en_US.UTF-8
- git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH git://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra
- - run:
- name: Build Cassandra
- command: |
- export LANG=en_US.UTF-8
- export JAVA_TOOL_OPTIONS="-Dfile.encoding=UTF8"
- export PATH=$PATH:$ANT_HOME/bin:$JAVA_HOME/bin
- cd ~/cassandra
- # Loop to prevent failure due to maven-ant-tasks not downloading a jar..
- for x in $(seq 1 3); do
- ${ANT_HOME}/bin/ant clean jar
- RETURN="$?"
- if [ "${RETURN}" -eq "0" ]; then
- break
- fi
- done
- # Exit, if we didn't build successfully
- if [ "${RETURN}" -ne "0" ]; then
- echo "Build failed with exit code: ${RETURN}"
- exit ${RETURN}
- fi
- no_output_timeout: 15m
- - run:
- name: Run eclipse-warnings
- command: |
- export LANG=en_US.UTF-8
- export JAVA_TOOL_OPTIONS="-Dfile.encoding=UTF8"
- export PATH=$PATH:$ANT_HOME/bin:$JAVA_HOME/bin
- cd ~/cassandra
- ant eclipse-warnings
- - persist_to_workspace:
- root: /home/cassandra
- paths:
- - cassandra
- - .m2
- unit_tests:
- <<: *env_settings
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Clone Cassandra Repository (via git)
+ command: |
+ git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH git://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra
+ - run:
+ name: Build Cassandra
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ cd ~/cassandra
+ # Loop to prevent failure due to maven-ant-tasks not downloading a jar..
+ for x in $(seq 1 3); do
+ ${ANT_HOME}/bin/ant clean jar
+ RETURN="$?"
+ if [ "${RETURN}" -eq "0" ]; then
+ break
+ fi
+ done
+ # Exit, if we didn't build successfully
+ if [ "${RETURN}" -ne "0" ]; then
+ echo "Build failed with exit code: ${RETURN}"
+ exit ${RETURN}
+ fi
+ no_output_timeout: 15m
+ - run:
+ name: Run eclipse-warnings
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ cd ~/cassandra
+ ant eclipse-warnings
+ - persist_to_workspace:
+ root: /home/cassandra
+ paths:
+ - cassandra
+ - .m2
+ environment:
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_unit_tests:
++ j8_dtests-no-vnodes:
+ docker:
+ - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
- name: Determine Unit Tests to Run
- command: |
- # reminder: this code (along with all the steps) is independently executed on every circle container
- # so the goal here is to get the circleci script to return the tests *this* container will run
- # which we do via the `circleci` cli tool.
-
- rm -fr ~/cassandra-dtest/upgrade_tests
- echo "***java tests***"
-
- # get all of our unit test filenames
- set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
-
- # split up the unit tests into groups based on the number of containers we have
- set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
- set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | cut -c 37-1000000 | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
- echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
- cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
- no_output_timeout: 15m
- - run:
- name: Log Environment Information
++ name: Clone Cassandra dtest Repository (via git)
+ command: |
- echo '*** id ***'
- id
- echo '*** cat /proc/cpuinfo ***'
- cat /proc/cpuinfo
- echo '*** free -m ***'
- free -m
- echo '*** df -m ***'
- df -m
- echo '*** ifconfig -a ***'
- ifconfig -a
- echo '*** uname -a ***'
- uname -a
- echo '*** mount ***'
- mount
- echo '*** env ***'
- env
- echo '*** java ***'
- which java
- java -version
++ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
- name: Run Unit Tests (testclasslist)
++ name: Configure virtualenv and python Dependencies
+ command: |
++ # note, this should be super quick as all dependencies should be pre-installed in the docker image
++ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
++ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
++ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
++ source ~/env/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- ant testclasslist -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
++ pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
++ pip3 freeze
++ - run:
++ name: Determine Tests to Run (j8_without_vnodes)
++ no_output_timeout: 5m
++ command: "# reminder: this code (along with all the steps) is independently\
++ \ executed on every circle container\n# so the goal here is to get the circleci\
++ \ script to return the tests *this* container will run\n# which we do via\
++ \ the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\n\
++ export PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\n\
++ echo \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail &&\
++ \ ./run_dtests.py --skip-resource-intensive-tests --dtest-print-tests-only\
++ \ --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw\
++ \ --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw\
++ \ /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw\
++ \ > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match\
++ \ any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci\
++ \ tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes\
++ \ > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt\
++ \ | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\
++ cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n"
++ - run:
++ name: Run dtests (j8_without_vnodes)
+ no_output_timeout: 15m
++ command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\n\
++ cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env/bin/activate\n\
++ export PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\n\
++ java -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env:\
++ \ $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need\
++ \ the \"set -o pipefail\" here so that the exit code that circleci will\
++ \ actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat\
++ \ /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nset -o pipefail &&\
++ \ cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"\
++ INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml\
++ \ -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS\
++ \ 2>&1 | tee /tmp/dtest/stdout.txt\n"
+ - store_test_results:
- path: /tmp/cassandra/build/test/output/
++ path: /tmp/results
+ - store_artifacts:
- path: /tmp/cassandra/build/test/output
- destination: junitxml
++ path: /tmp/dtest
++ destination: dtest_j8_without_vnodes
+ - store_artifacts:
- path: /tmp/cassandra/build/test/logs
- destination: logs
++ path: ~/cassandra-dtest/logs
++ destination: dtest_j8_without_vnodes_logs
+ environment:
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_jvm_dtests:
++ j8_upgradetests-no-vnodes:
docker:
- - image: *docker_image
- environment:
- <<: *env_vars
+ - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
- parallelism: 1
++ parallelism: 4
steps:
- - attach_workspace:
- at: /home/cassandra
- - run:
- name: Determine Tests to Run
- no_output_timeout: 15m
- command: |
- # reminder: this code (along with all the steps) is independently executed on every circle container
- # so the goal here is to get the circleci script to return the tests *this* container will run
- # which we do via the `circleci` cli tool.
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
- name: Run Unit Tests (test-jvm-dtest-forking)
++ name: Clone Cassandra dtest Repository (via git)
++ command: |
++ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
++ - run:
++ name: Configure virtualenv and python Dependencies
+ command: |
++ # note, this should be super quick as all dependencies should be pre-installed in the docker image
++ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
++ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
++ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
++ source ~/env/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- ant clean test-jvm-dtest-forking
++ pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
++ pip3 freeze
++ - run:
++ name: Determine Tests to Run (j8_upgradetests_without_vnodes)
++ no_output_timeout: 5m
++ command: |
++ # reminder: this code (along with all the steps) is independently executed on every circle container
++ # so the goal here is to get the circleci script to return the tests *this* container will run
++ # which we do via the `circleci` cli tool.
+
- export LANG=en_US.UTF-8
- rm -fr ~/cassandra-dtest/upgrade_tests
- echo "***java tests***"
++ cd cassandra-dtest
++ source ~/env/bin/activate
++ export PATH=$JAVA_HOME/bin:$PATH
+
- # get all of our unit test filenames
- set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
++ if [ -n 'RUN_STATIC_UPGRADE_MATRIX=true' ]; then
++ export RUN_STATIC_UPGRADE_MATRIX=true
++ fi
+
- # split up the unit tests into groups based on the number of containers we have
- set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
- set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | cut -c 37-1000000 | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
- echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
- cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
++ echo "***Collected DTests (j8_upgradetests_without_vnodes)***"
++ set -eo pipefail && ./run_dtests.py --execute-upgrade-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw --cassandra-dir=../cassandra
++ if [ -z '^upgrade_tests' ]; then
++ mv /tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw /tmp/all_dtest_tests_j8_upgradetests_without_vnodes
++ else
++ grep -e '^upgrade_tests' /tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw > /tmp/all_dtest_tests_j8_upgradetests_without_vnodes || { echo "Filter did not match any tests! Exiting build."; exit 0; }
++ fi
++ set -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_upgradetests_without_vnodes > /tmp/split_dtest_tests_j8_upgradetests_without_vnodes.txt
++ cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes.txt | tr '\n' ' ' > /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
++ cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
++ - run:
++ name: Run dtests (j8_upgradetests_without_vnodes)
+ no_output_timeout: 15m
++ command: |
++ echo "cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt"
++ cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
+
- - run:
- name: Run Unit Tests
- command: |
- export LANG=en_US.UTF-8
- export JAVA_TOOL_OPTIONS="-Dfile.encoding=UTF8"
- export PATH=$PATH:$ANT_HOME/bin:$JAVA_HOME/bin
- #Skip all syncing to disk to avoid performance issues in flaky CI environments
- export CASSANDRA_SKIP_SYNC=true
++ source ~/env/bin/activate
++ export PATH=$JAVA_HOME/bin:$PATH
++ if [ -n 'RUN_STATIC_UPGRADE_MATRIX=true' ]; then
++ export RUN_STATIC_UPGRADE_MATRIX=true
++ fi
+
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- ant testclasslist -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
++ java -version
++ cd ~/cassandra-dtest
++ mkdir -p /tmp/dtest
+
- if [ "${CIRCLE_NODE_INDEX}" -eq "1" ]; then
- ant test-jvm-dtest-forking
- fi
- no_output_timeout: 15m
- - store_test_results:
- path: /tmp/cassandra/build/test/output/
- - store_artifacts:
- path: /tmp/cassandra/build/test/output
- destination: junitxml
- - store_artifacts:
- path: /tmp/cassandra/build/test/logs
- destination: logs
- dtests-with-vnodes:
- <<: *env_settings
++ echo "env: $(env)"
++ echo "** done env"
++ mkdir -p /tmp/results/dtests
++ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
++ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt`
++ set -o pipefail && cd ~/cassandra-dtest && pytest --execute-upgrade-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_upgradetests_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
- path: /tmp/cassandra/build/test/output/
++ path: /tmp/results
+ - store_artifacts:
- path: /tmp/cassandra/build/test/output
- destination: junitxml
++ path: /tmp/dtest
++ destination: dtest_j8_upgradetests_without_vnodes
+ - store_artifacts:
- path: /tmp/cassandra/build/test/logs
- destination: logs
++ path: ~/cassandra-dtest/logs
++ destination: dtest_j8_upgradetests_without_vnodes_logs
+ environment:
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- utests_long:
++ utests_stress:
+ docker:
+ - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
- name: Run Unit Tests (long-test)
++ name: Run Unit Tests (stress-test)
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
- ant clean long-test
++ ant clean stress-test
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- utests_compression:
++ j8_unit_tests:
docker:
- - image: *docker_image
- environment:
- <<: *env_vars
+ - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
steps:
- - attach_workspace:
- at: /home/cassandra
- - run:
- name: Clone Cassandra dtest Repository (via git)
- command: |
- export LANG=en_US.UTF-8
- git clone --single-branch --branch master --depth 1 git://github.com/apache/cassandra-dtest.git ~/cassandra-dtest
- - run:
- name: Configure virtualenv and python Dependencies
- command: |
- # note, this should be super quick as all dependencies should be pre-installed in the docker image
- # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
- # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
- # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
- export LANG=en_US.UTF-8
- source ~/env/bin/activate
- export PATH=$PATH:$ANT_HOME/bin:$JAVA_HOME/bin
- export CASS_DRIVER_NO_EXTENSIONS=true
- export CASS_DRIVER_NO_CYTHON=true
- pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
- pip3 freeze
- - run:
- name: Determine Tests to Run
- no_output_timeout: 5m
- command: |
- # reminder: this code (along with all the steps) is independently executed on every circle container
- # so the goal here is to get the circleci script to return the tests *this* container will run
- # which we do via the `circleci` cli tool.
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine Unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
- export LANG=en_US.UTF-8
- cd cassandra-dtest
- source ~/env/bin/activate
- export PATH=$PATH:$ANT_HOME/bin:$JAVA_HOME/bin
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
- echo "***Collected DTests (with vnodes)***"
- set -eo pipefail && ./run_dtests.py --use-vnodes --dtest-print-tests-only --skip-resource-intensive-tests --dtest-print-tests-output=/tmp/all_dtest_tests_with_vnodes --cassandra-dir="../cassandra"
- set -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_with_vnodes > /tmp/split_dtest_tests_with_vnodes.txt
- cat /tmp/split_dtest_tests_with_vnodes.txt | tr '\n' ' ' > /tmp/split_dtest_tests_with_vnodes_final.txt
- # cat /tmp/split_dtest_tests_with_vnodes.txt
- cat /tmp/split_dtest_tests_with_vnodes_final.txt
- - run:
- name: Run dtests (with vnodes)
- no_output_timeout: 15m
- command: |
- echo "cat /tmp/split_dtest_tests_with_vnodes_final.txt"
- cat /tmp/split_dtest_tests_with_vnodes_final.txt
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
- source ~/env/bin/activate
- export PATH=$PATH:$ANT_HOME/bin:$JAVA_HOME/bin
-
- cd ~/cassandra-dtest
- mkdir -p /tmp/dtest
-
- echo "env: $(env)"
- echo "** done env"
- mkdir -p /tmp/results/dtests
- # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
- export SPLIT_TESTS=`cat /tmp/split_dtest_tests_with_vnodes_final.txt`
- #Skip all syncing to disk to avoid performance issues in flaky CI environments
- export CASSANDRA_SKIP_SYNC=true
- set -o pipefail && cd ~/cassandra-dtest && pytest --log-level="INFO" --use-vnodes --num-tokens=32 --junit-xml=/tmp/results/dtests/pytest_result_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --skip-resource-intensive-tests --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
- - store_test_results:
- path: /tmp/results
- - store_artifacts:
- path: /tmp/dtest
- destination: dtest_with_vnodes
- - store_artifacts:
- path: ~/cassandra-dtest/logs
- destination: dtest_with_vnodes_logs
- dtests-no-vnodes:
- <<: *env_settings
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | cut -c 37-1000000 | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
- name: Run Unit Tests (testclasslist-compression)
++ name: Run Unit Tests (testclasslist)
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
- ant testclasslist-compression -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
++ ant testclasslist -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_dtests-with-vnodes:
+ docker:
+ - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently\
+ \ executed on every circle container\n# so the goal here is to get the circleci\
+ \ script to return the tests *this* container will run\n# which we do via\
+ \ the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\n\
+ export PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\n\
+ echo \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py\
+ \ --use-vnodes --skip-resource-intensive-tests --dtest-print-tests-only\
+ \ --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\n\
+ if [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\n\
+ else\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes\
+ \ || { echo \"Filter did not match any tests! Exiting build.\"; exit 0;\
+ \ }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname\
+ \ /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\n\
+ cat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\
+ cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j8_with_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat\
+ \ /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env/bin/activate\n\
+ export PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\n\
+ java -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env:\
+ \ $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need\
+ \ the \"set -o pipefail\" here so that the exit code that circleci will\
+ \ actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat\
+ \ /tmp/split_dtest_tests_j8_with_vnodes_final.txt`\nset -o pipefail && cd\
+ \ ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests\
+ \ --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml\
+ \ -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS\
+ \ 2>&1 | tee /tmp/dtest/stdout.txt\n"
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j8_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j8_with_vnodes_logs
+ environment:
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_dtests-no-vnodes:
++ j8_jvm_dtests:
docker:
- - image: *docker_image
- environment:
- <<: *env_vars
+ - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
- parallelism: 4
++ parallelism: 1
steps:
- - attach_workspace:
- at: /home/cassandra
- - run:
- name: Clone Cassandra dtest Repository (via git)
- command: |
- export LANG=en_US.UTF-8
- git clone --single-branch --branch master --depth 1 git://github.com/apache/cassandra-dtest.git ~/cassandra-dtest
- - run:
- name: Configure virtualenv and python Dependencies
- command: |
- # note, this should be super quick as all dependencies should be pre-installed in the docker image
- # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
- # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
- # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
- export LANG=en_US.UTF-8
- source ~/env/bin/activate
- export PATH=$PATH:$ANT_HOME/bin:$JAVA_HOME/bin
- export CASS_DRIVER_NO_EXTENSIONS=true
- export CASS_DRIVER_NO_CYTHON=true
- pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
- pip3 freeze
- - run:
- name: Determine Tests to Run
- no_output_timeout: 5m
- command: |
- # reminder: this code (along with all the steps) is independently executed on every circle container
- # so the goal here is to get the circleci script to return the tests *this* container will run
- # which we do via the `circleci` cli tool.
-
- export LANG=en_US.UTF-8
- cd cassandra-dtest
- source ~/env/bin/activate
- export PATH=$PATH:$ANT_HOME/bin:$JAVA_HOME/bin
-
- echo "***Collected DTests (without vnodes)***"
- ./run_dtests.py --dtest-print-tests-only --skip-resource-intensive-tests --dtest-print-tests-output=/tmp/all_dtest_tests_without_vnodes --cassandra-dir="../cassandra"
- set -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_without_vnodes > /tmp/split_dtest_tests_without_vnodes.txt
- cat /tmp/split_dtest_tests_without_vnodes.txt | tr '\n' ' ' > /tmp/split_dtest_tests_without_vnodes_final.txt
- # cat /tmp/split_dtest_tests_without_vnodes.txt
- cat /tmp/split_dtest_tests_without_vnodes_final.txt
- - run:
- name: Run dtests (without vnodes)
- no_output_timeout: 15m
- command: |
- # for now require at least 50 circleci containers to run the dtests (with less resources the tests won't produce reliable results or will fail to complete)
- if [ $CIRCLE_NODE_TOTAL -gt 0 ]; then
- source ~/env/bin/activate
- export PATH=$PATH:$ANT_HOME/bin:$JAVA_HOME/bin
-
- cd ~/cassandra-dtest
- mkdir -p /tmp/dtest
-
- mkdir -p /tmp/results/dtests
- # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
- export SPLIT_TESTS=`cat /tmp/split_dtest_tests_without_vnodes_final.txt`
- #Skip all syncing to disk to avoid performance issues in flaky CI environments
- export CASSANDRA_SKIP_SYNC=true
- set -o pipefail && cd ~/cassandra-dtest && pytest --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_novnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --skip-resource-intensive-tests --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout-novnodes.txt
- fi
- - store_test_results:
- path: /tmp/results
- - store_artifacts:
- path: /tmp/dtest
- destination: dtest_no_vnodes
- - store_artifacts:
- path: ~/cassandra-dtest/logs
- destination: dtest_no_vnodes_logs
- upgrade_tests:
- <<: *env_settings
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
- name: Clone Cassandra dtest Repository (via git)
- command: |
- git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- - run:
- name: Configure virtualenv and python Dependencies
++ name: Run Unit Tests (test-jvm-dtest-forking)
+ command: |
- # note, this should be super quick as all dependencies should be pre-installed in the docker image
- # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
- # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
- # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
- source ~/env/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
- pip3 freeze
- - run:
- name: Determine Tests to Run (j8_without_vnodes)
- no_output_timeout: 5m
- command: "# reminder: this code (along with all the steps) is independently\
- \ executed on every circle container\n# so the goal here is to get the circleci\
- \ script to return the tests *this* container will run\n# which we do via\
- \ the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\n\
- export PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\n\
- echo \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail &&\
- \ ./run_dtests.py --skip-resource-intensive-tests --dtest-print-tests-only\
- \ --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw\
- \ --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw\
- \ /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw\
- \ > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match\
- \ any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci\
- \ tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes\
- \ > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt\
- \ | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\
- cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n"
- - run:
- name: Run dtests (j8_without_vnodes)
++ time mv ~/cassandra /tmp
++ cd /tmp/cassandra
++ ant clean test-jvm-dtest-forking
+ no_output_timeout: 15m
- command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\n\
- cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env/bin/activate\n\
- export PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\n\
- java -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env:\
- \ $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need\
- \ the \"set -o pipefail\" here so that the exit code that circleci will\
- \ actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat\
- \ /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nset -o pipefail &&\
- \ cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"\
- INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml\
- \ -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS\
- \ 2>&1 | tee /tmp/dtest/stdout.txt\n"
+ - store_test_results:
- path: /tmp/results
++ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
- path: /tmp/dtest
- destination: dtest_j8_without_vnodes
++ path: /tmp/cassandra/build/test/output
++ destination: junitxml
+ - store_artifacts:
- path: ~/cassandra-dtest/logs
- destination: dtest_j8_without_vnodes_logs
++ path: /tmp/cassandra/build/test/logs
++ destination: logs
+ environment:
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_upgradetests-no-vnodes:
++ utests_long:
+ docker:
+ - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
- parallelism: 4
++ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
- name: Clone Cassandra dtest Repository (via git)
- command: |
- git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- - run:
- name: Configure virtualenv and python Dependencies
++ name: Run Unit Tests (long-test)
+ command: |
- # note, this should be super quick as all dependencies should be pre-installed in the docker image
- # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
- # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
- # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
- source ~/env/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
- pip3 freeze
++ time mv ~/cassandra /tmp
++ cd /tmp/cassandra
++ ant clean long-test
++ no_output_timeout: 15m
++ - store_test_results:
++ path: /tmp/cassandra/build/test/output/
++ - store_artifacts:
++ path: /tmp/cassandra/build/test/output
++ destination: junitxml
++ - store_artifacts:
++ path: /tmp/cassandra/build/test/logs
++ destination: logs
++ environment:
++ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ - ANT_HOME: /usr/share/ant
++ - LANG: en_US.UTF-8
++ - KEEP_TEST_DIR: true
++ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
++ - PYTHONIOENCODING: utf-8
++ - PYTHONUNBUFFERED: true
++ - CASS_DRIVER_NO_EXTENSIONS: true
++ - CASS_DRIVER_NO_CYTHON: true
++ - CASSANDRA_SKIP_SYNC: true
++ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
++ - DTEST_BRANCH: master
++ - CCM_MAX_HEAP_SIZE: 1024M
++ - CCM_HEAP_NEWSIZE: 256M
++ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ utests_compression:
+ docker:
- - image: *docker_image
- environment:
- <<: *env_vars
++ - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
++ resource_class: medium
++ working_directory: ~/
++ shell: /bin/bash -eo pipefail -l
++ parallelism: 4
+ steps:
- - attach_workspace:
- at: /home/cassandra
- - run:
- name: Clone Cassandra dtest Repository (via git)
- command: |
- export LANG=en_US.UTF-8
- git clone --single-branch --branch master --depth 1 git://github.com/apache/cassandra-dtest.git ~/cassandra-dtest
- - run:
- name: Configure virtualenv and python Dependencies
- command: |
- # note, this should be super quick as all dependencies should be pre-installed in the docker image
- # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
- # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
- # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
- export LANG=en_US.UTF-8
- source ~/env/bin/activate
- export PATH=$PATH:$ANT_HOME/bin:$JAVA_HOME/bin
- export CASS_DRIVER_NO_EXTENSIONS=true
- export CASS_DRIVER_NO_CYTHON=true
- pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
- pip3 freeze
- - run:
- name: Determine Tests to Run
- no_output_timeout: 5m
- command: |
- # reminder: this code (along with all the steps) is independently executed on every circle container
- # so the goal here is to get the circleci script to return the tests *this* container will run
- # which we do via the `circleci` cli tool.
++ - attach_workspace:
++ at: /home/cassandra
+ - run:
- name: Determine Tests to Run (j8_upgradetests_without_vnodes)
- no_output_timeout: 5m
++ name: Determine Unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
- export LANG=en_US.UTF-8
- export RUN_STATIC_UPGRADE_MATRIX=true
- cd cassandra-dtest
- source ~/env/bin/activate
- export PATH=$PATH:$ANT_HOME/bin:$JAVA_HOME/bin
- cd cassandra-dtest
- source ~/env/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
++ rm -fr ~/cassandra-dtest/upgrade_tests
++ echo "***java tests***"
- echo "***Collected upgrade DTests (without vnodes)***"
- ./run_dtests.py --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_without_vnodes --cassandra-dir="../cassandra" --execute-upgrade-tests
- cat /tmp/all_dtest_tests_without_vnodes | grep '^upgrade_tests' > /tmp/all_upgrade_dtest_tests_without_vnodes
- set -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_upgrade_dtest_tests_without_vnodes > /tmp/split_upgrade_dtest_tests_without_vnodes.txt
- cat /tmp/split_upgrade_dtest_tests_without_vnodes.txt | tr '\n' ' ' > /tmp/split_upgrade_dtest_tests_without_vnodes_final.txt
- cat /tmp/split_upgrade_dtest_tests_without_vnodes_final.txt
- - run:
- name: Run upgrade dtests (without vnodes)
- no_output_timeout: 15m
- command: |
- # for now require at least 50 circleci containers to run the dtests (with less resources the tests won't produce reliable results or will fail to complete)
- if [ $CIRCLE_NODE_TOTAL -gt 0 ]; then
- source ~/env/bin/activate
- export PATH=$PATH:$ANT_HOME/bin:$JAVA_HOME/bin
- export RUN_STATIC_UPGRADE_MATRIX=true
- cd ~/cassandra-dtest
- mkdir -p /tmp/dtest
- if [ -n 'RUN_STATIC_UPGRADE_MATRIX=true' ]; then
- export RUN_STATIC_UPGRADE_MATRIX=true
- fi
++ # get all of our unit test filenames
++ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
- echo "***Collected DTests (j8_upgradetests_without_vnodes)***"
- set -eo pipefail && ./run_dtests.py --execute-upgrade-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw --cassandra-dir=../cassandra
- if [ -z '^upgrade_tests' ]; then
- mv /tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw /tmp/all_dtest_tests_j8_upgradetests_without_vnodes
- else
- grep -e '^upgrade_tests' /tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw > /tmp/all_dtest_tests_j8_upgradetests_without_vnodes || { echo "Filter did not match any tests! Exiting build."; exit 0; }
- fi
- set -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_upgradetests_without_vnodes > /tmp/split_dtest_tests_j8_upgradetests_without_vnodes.txt
- cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes.txt | tr '\n' ' ' > /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
- cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
- - run:
- name: Run dtests (j8_upgradetests_without_vnodes)
++ # split up the unit tests into groups based on the number of containers we have
++ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
++ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | cut -c 37-1000000 | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
++ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
++ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
++ - run:
++ name: Log Environment Information
+ command: |
- echo "cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt"
- cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
-
- source ~/env/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
- if [ -n 'RUN_STATIC_UPGRADE_MATRIX=true' ]; then
- export RUN_STATIC_UPGRADE_MATRIX=true
- fi
-
++ echo '*** id ***'
++ id
++ echo '*** cat /proc/cpuinfo ***'
++ cat /proc/cpuinfo
++ echo '*** free -m ***'
++ free -m
++ echo '*** df -m ***'
++ df -m
++ echo '*** ifconfig -a ***'
++ ifconfig -a
++ echo '*** uname -a ***'
++ uname -a
++ echo '*** mount ***'
++ mount
++ echo '*** env ***'
++ env
++ echo '*** java ***'
++ which java
+ java -version
- cd ~/cassandra-dtest
- mkdir -p /tmp/dtest
-
- echo "env: $(env)"
- echo "** done env"
- mkdir -p /tmp/results/dtests
- # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
- export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt`
- set -o pipefail && cd ~/cassandra-dtest && pytest --execute-upgrade-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_upgradetests_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
++ - run:
++ name: Run Unit Tests (testclasslist-compression)
++ command: |
++ export PATH=$JAVA_HOME/bin:$PATH
++ time mv ~/cassandra /tmp
++ cd /tmp/cassandra
++ ant testclasslist-compression -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
++ no_output_timeout: 15m
+ - store_test_results:
- path: /tmp/results
++ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
- path: /tmp/dtest
- destination: dtest_j8_upgradetests_without_vnodes
++ path: /tmp/cassandra/build/test/output
++ destination: junitxml
+ - store_artifacts:
- path: ~/cassandra-dtest/logs
- destination: dtest_j8_upgradetests_without_vnodes_logs
++ path: /tmp/cassandra/build/test/logs
++ destination: logs
+ environment:
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ workflows:
+ version: 2
+ build_and_run_tests:
+ jobs:
+ - build
+ - j8_unit_tests:
+ requires:
+ - build
+ - j8_jvm_dtests:
+ requires:
+ - build
+ - start_utests_long:
+ type: approval
+ requires:
+ - build
+ - utests_long:
+ requires:
+ - start_utests_long
+ - start_utests_compression:
+ type: approval
+ requires:
+ - build
+ - utests_compression:
+ requires:
+ - start_utests_compression
++ - start_utests_stress:
++ type: approval
++ requires:
++ - build
++ - utests_stress:
++ requires:
++ - start_utests_stress
+ - start_j8_dtests:
+ type: approval
+ requires:
+ - build
+ - j8_dtests-with-vnodes:
+ requires:
+ - start_j8_dtests
+ - j8_dtests-no-vnodes:
+ requires:
+ - start_j8_dtests
+ - start_upgrade_tests:
+ type: approval
+ requires:
+ - build
+ - j8_upgradetests-no-vnodes:
+ requires:
+ - start_upgrade_tests
- mkdir -p /tmp/results/dtests
- # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
- export SPLIT_TESTS=`cat /tmp/split_upgrade_dtest_tests_without_vnodes_final.txt`
- #Skip all syncing to disk to avoid performance issues in flaky CI environments
- export CASSANDRA_SKIP_SYNC=true
- set -o pipefail && cd ~/cassandra-dtest && pytest --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_novnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --execute-upgrade-tests --keep-test-dir --log-cli-level=debug $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout-novnodes.txt
- fi
- - store_test_results:
- path: /tmp/results
- - store_artifacts:
- path: /tmp/dtest
- destination: dtest_upgrade_no_vnodes
- - store_artifacts:
- path: ~/cassandra-dtest/logs
- destination: dtest_upgrade_no_vnodes_logs
+ # Original config.yml file:
+ # version: 2.1
+ #
+ # default_env_vars: &default_env_vars
+ # JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ # ANT_HOME: /usr/share/ant
+ # LANG: en_US.UTF-8
+ # KEEP_TEST_DIR: true
+ # DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ # PYTHONIOENCODING: utf-8
+ # PYTHONUNBUFFERED: true
+ # CASS_DRIVER_NO_EXTENSIONS: true
+ # CASS_DRIVER_NO_CYTHON: true
+ # #Skip all syncing to disk to avoid performance issues in flaky CI environments
+ # CASSANDRA_SKIP_SYNC: true
+ # DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ # DTEST_BRANCH: master
+ # CCM_MAX_HEAP_SIZE: 1024M
+ # CCM_HEAP_NEWSIZE: 256M
+ #
+ # j8_par_executor: &j8_par_executor
+ # executor:
+ # name: java8-executor
+ # #exec_resource_class: xlarge
+ # parallelism: 4
+ #
+ # j8_seq_executor: &j8_seq_executor
+ # executor:
+ # name: java8-executor
+ # #exec_resource_class: xlarge
+ # parallelism: 1 # sequential, single container tests: no parallelism benefits
+ #
+ # with_dtests_jobs: &with_dtest_jobs
+ # jobs:
+ # - build
+ # # Java 8 unit tests will be run automatically
+ # - j8_unit_tests:
+ # requires:
+ # - build
+ # - j8_jvm_dtests:
+ # requires:
+ # - build
+ # # specialized unit tests (all run on request using Java 8)
+ # - start_utests_long:
+ # type: approval
+ # requires:
+ # - build
+ # - utests_long:
+ # requires:
+ # - start_utests_long
+ # - start_utests_compression:
+ # type: approval
+ # requires:
+ # - build
+ # - utests_compression:
+ # requires:
+ # - start_utests_compression
++# - start_utests_stress:
++# type: approval
++# requires:
++# - build
++# - utests_stress:
++# requires:
++# - start_utests_stress
+ # # Java 8 dtests (on request)
+ # - start_j8_dtests:
+ # type: approval
+ # requires:
+ # - build
+ # - j8_dtests-with-vnodes:
+ # requires:
+ # - start_j8_dtests
+ # - j8_dtests-no-vnodes:
+ # requires:
+ # - start_j8_dtests
+ # # Java 8 upgrade tests
+ # - start_upgrade_tests:
+ # type: approval
+ # requires:
+ # - build
+ # - j8_upgradetests-no-vnodes:
+ # requires:
+ # - start_upgrade_tests
+ #
+ # with_dtest_jobs_only: &with_dtest_jobs_only
+ # jobs:
+ # - build
+ # - j8_dtests-with-vnodes:
+ # requires:
+ # - build
+ # - j8_dtests-no-vnodes:
+ # requires:
+ # - build
+ #
+ # workflows:
+ # version: 2
+ # build_and_run_tests: *with_dtest_jobs
+ # #build_and_run_tests: *with_dtest_jobs_only
+ #
+ # executors:
+ # java8-executor:
+ # parameters:
+ # exec_resource_class:
+ # type: string
+ # default: medium
+ # docker:
+ # - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ # resource_class: << parameters.exec_resource_class >>
+ # working_directory: ~/
+ # shell: /bin/bash -eo pipefail -l
+ # environment:
+ # <<: *default_env_vars
+ # JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ # JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ #
+ # jobs:
+ # build:
+ # executor: java8-executor
+ # parallelism: 1 # This job doesn't benefit from parallelism
+ # steps:
+ # - log_environment
+ # - clone_cassandra
+ # - build_cassandra
+ # - run_eclipse_warnings
+ # - persist_to_workspace:
+ # root: /home/cassandra
+ # paths:
+ # - cassandra
+ # - .m2
+ #
+ # j8_unit_tests:
+ # <<: *j8_par_executor
+ # steps:
+ # - attach_workspace:
+ # at: /home/cassandra
+ # - create_junit_containers
+ # - log_environment
+ # - run_parallel_junit_tests
+ #
+ # j8_jvm_dtests:
+ # <<: *j8_seq_executor
+ # steps:
+ # - attach_workspace:
+ # at: /home/cassandra
+ # - run_junit_tests:
+ # target: test-jvm-dtest-forking
+ #
+ # utests_long:
+ # <<: *j8_seq_executor
+ # steps:
+ # - attach_workspace:
+ # at: /home/cassandra
+ # - run_junit_tests:
+ # target: long-test
+ #
+ # utests_compression:
+ # <<: *j8_par_executor
+ # steps:
+ # - attach_workspace:
+ # at: /home/cassandra
+ # - create_junit_containers
+ # - log_environment
+ # - run_parallel_junit_tests:
+ # target: testclasslist-compression
+ #
++# utests_stress:
++# <<: *j8_seq_executor
++# steps:
++# - attach_workspace:
++# at: /home/cassandra
++# - run_junit_tests:
++# target: stress-test
++#
+ # j8_dtests-with-vnodes:
+ # <<: *j8_par_executor
+ # steps:
+ # - attach_workspace:
+ # at: /home/cassandra
+ # - clone_dtest
+ # - create_venv
+ # - create_dtest_containers:
+ # file_tag: j8_with_vnodes
+ # run_dtests_extra_args: '--use-vnodes --skip-resource-intensive-tests'
+ # - run_dtests:
+ # file_tag: j8_with_vnodes
+ # pytest_extra_args: '--use-vnodes --num-tokens=32 --skip-resource-intensive-tests'
+ #
+ # j8_dtests-no-vnodes:
+ # <<: *j8_par_executor
+ # steps:
+ # - attach_workspace:
+ # at: /home/cassandra
+ # - clone_dtest
+ # - create_venv
+ # - create_dtest_containers:
+ # file_tag: j8_without_vnodes
+ # run_dtests_extra_args: '--skip-resource-intensive-tests'
+ # - run_dtests:
+ # file_tag: j8_without_vnodes
+ # pytest_extra_args: '--skip-resource-intensive-tests'
+ #
+ # j8_upgradetests-no-vnodes:
+ # <<: *j8_par_executor
+ # steps:
+ # - attach_workspace:
+ # at: /home/cassandra
+ # - clone_dtest
+ # - create_venv
+ # - create_dtest_containers:
+ # file_tag: j8_upgradetests_without_vnodes
+ # run_dtests_extra_args: '--execute-upgrade-tests'
+ # extra_env_args: 'RUN_STATIC_UPGRADE_MATRIX=true'
+ # tests_filter_pattern: '^upgrade_tests'
+ # - run_dtests:
+ # file_tag: j8_upgradetests_without_vnodes
+ # extra_env_args: 'RUN_STATIC_UPGRADE_MATRIX=true'
+ # pytest_extra_args: '--execute-upgrade-tests'
+ #
+ # commands:
+ # log_environment:
+ # steps:
+ # - run:
+ # name: Log Environment Information
+ # command: |
+ # echo '*** id ***'
+ # id
+ # echo '*** cat /proc/cpuinfo ***'
+ # cat /proc/cpuinfo
+ # echo '*** free -m ***'
+ # free -m
+ # echo '*** df -m ***'
+ # df -m
+ # echo '*** ifconfig -a ***'
+ # ifconfig -a
+ # echo '*** uname -a ***'
+ # uname -a
+ # echo '*** mount ***'
+ # mount
+ # echo '*** env ***'
+ # env
+ # echo '*** java ***'
+ # which java
+ # java -version
+ #
+ # clone_cassandra:
+ # steps:
+ # - run:
+ # name: Clone Cassandra Repository (via git)
+ # command: |
+ # git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH git://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra
+ #
+ # clone_dtest:
+ # steps:
+ # - run:
+ # name: Clone Cassandra dtest Repository (via git)
+ # command: |
+ # git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ #
+ # build_cassandra:
+ # steps:
+ # - run:
+ # name: Build Cassandra
+ # command: |
+ # export PATH=$JAVA_HOME/bin:$PATH
+ # cd ~/cassandra
+ # # Loop to prevent failure due to maven-ant-tasks not downloading a jar..
+ # for x in $(seq 1 3); do
+ # ${ANT_HOME}/bin/ant clean jar
+ # RETURN=\"$?\"
+ # if [ \"${RETURN}\" -eq \"0\" ]; then
+ # break
+ # fi
+ # done
+ # # Exit, if we didn't build successfully
+ # if [ \"${RETURN}\" -ne \"0\" ]; then
+ # echo \"Build failed with exit code: ${RETURN}\"
+ # exit ${RETURN}
+ # fi
+ # no_output_timeout: 15m
+ #
+ # run_eclipse_warnings:
+ # steps:
+ # - run:
+ # name: Run eclipse-warnings
+ # command: |
+ # export PATH=$JAVA_HOME/bin:$PATH
+ # cd ~/cassandra
+ # ant eclipse-warnings
+ #
+ # create_junit_containers:
+ # steps:
+ # - run:
+ # name: Determine Unit Tests to Run
+ # command: |
+ # # reminder: this code (along with all the steps) is independently executed on every circle container
+ # # so the goal here is to get the circleci script to return the tests *this* container will run
+ # # which we do via the `circleci` cli tool.
+ #
+ # rm -fr ~/cassandra-dtest/upgrade_tests
+ # echo \"***java tests***\"
+ #
+ # # get all of our unit test filenames
+ # set -eo pipefail && circleci tests glob \"$HOME/cassandra/test/unit/**/*.java\" > /tmp/all_java_unit_tests.txt
+ #
+ # # split up the unit tests into groups based on the number of containers we have
+ # set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ # set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | cut -c 37-1000000 | grep \"Test\\.java$\" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ # echo \"** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt\"
+ # cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ #
+ # no_output_timeout: 15m
+ #
+ # run_junit_tests:
+ # parameters:
+ # target:
+ # type: string
+ # no_output_timeout:
+ # type: string
+ # default: 15m
+ # steps:
+ # - run:
+ # name: Run Unit Tests (<<parameters.target>>)
+ # # Please note that we run `clean` and therefore rebuild the project, as we can't run tests on Java 8 in case
+ # # based on Java 11 builds.
+ # command: |
+ # export PATH=$JAVA_HOME/bin:$PATH
+ # time mv ~/cassandra /tmp
+ # cd /tmp/cassandra
+ # ant clean <<parameters.target>>
+ # no_output_timeout: <<parameters.no_output_timeout>>
+ # - store_test_results:
+ # path: /tmp/cassandra/build/test/output/
+ # - store_artifacts:
+ # path: /tmp/cassandra/build/test/output
+ # destination: junitxml
+ # - store_artifacts:
+ # path: /tmp/cassandra/build/test/logs
+ # destination: logs
+ #
+ # run_parallel_junit_tests:
+ # parameters:
+ # target:
+ # type: string
+ # default: testclasslist
+ # no_output_timeout:
+ # type: string
+ # default: 15m
+ # steps:
+ # - run:
+ # name: Run Unit Tests (<<parameters.target>>)
+ # # Please note that we run `clean` and therefore rebuild the project, as we can't run tests on Java 8 in case
+ # # based on Java 11 builds.
+ # command: |
+ # export PATH=$JAVA_HOME/bin:$PATH
+ # time mv ~/cassandra /tmp
+ # cd /tmp/cassandra
+ # ant <<parameters.target>> -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
+ # no_output_timeout: <<parameters.no_output_timeout>>
+ # - store_test_results:
+ # path: /tmp/cassandra/build/test/output/
+ # - store_artifacts:
+ # path: /tmp/cassandra/build/test/output
+ # destination: junitxml
+ # - store_artifacts:
+ # path: /tmp/cassandra/build/test/logs
+ # destination: logs
+ #
+ # create_venv:
+ # steps:
+ # - run:
+ # name: Configure virtualenv and python Dependencies
+ # command: |
+ # # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ # source ~/env/bin/activate
+ # export PATH=$JAVA_HOME/bin:$PATH
+ # pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
+ # pip3 freeze
+ #
+ # create_dtest_containers:
+ # parameters:
+ # file_tag:
+ # type: string
+ # run_dtests_extra_args:
+ # type: string
+ # default: ''
+ # extra_env_args:
+ # type: string
+ # default: ''
+ # tests_filter_pattern:
+ # type: string
+ # default: ''
+ # steps:
+ # - run:
+ # name: Determine Tests to Run (<<parameters.file_tag>>)
+ # no_output_timeout: 5m
+ # command: |
+ # # reminder: this code (along with all the steps) is independently executed on every circle container
+ # # so the goal here is to get the circleci script to return the tests *this* container will run
+ # # which we do via the `circleci` cli tool.
+ #
+ # cd cassandra-dtest
+ # source ~/env/bin/activate
+ # export PATH=$JAVA_HOME/bin:$PATH
+ #
+ # if [ -n '<<parameters.extra_env_args>>' ]; then
+ # export <<parameters.extra_env_args>>
+ # fi
+ #
+ # echo \"***Collected DTests (<<parameters.file_tag>>)***\"
+ # set -eo pipefail && ./run_dtests.py <<parameters.run_dtests_extra_args>> --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_<<parameters.file_tag>>_raw --cassandra-dir=../cassandra
+ # if [ -z '<<parameters.tests_filter_pattern>>' ]; then
+ # mv /tmp/all_dtest_tests_<<parameters.file_tag>>_raw /tmp/all_dtest_tests_<<parameters.file_tag>>
+ # else
+ # grep -e '<<parameters.tests_filter_pattern>>' /tmp/all_dtest_tests_<<parameters.file_tag>>_raw > /tmp/all_dtest_tests_<<parameters.file_tag>> || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }
+ # fi
+ # set -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_<<parameters.file_tag>> > /tmp/split_dtest_tests_<<parameters.file_tag>>.txt
+ # cat /tmp/split_dtest_tests_<<parameters.file_tag>>.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_<<parameters.file_tag>>_final.txt
+ # cat /tmp/split_dtest_tests_<<parameters.file_tag>>_final.txt
+ #
+ # run_dtests:
+ # parameters:
+ # file_tag:
+ # type: string
+ # pytest_extra_args:
+ # type: string
+ # default: ''
+ # extra_env_args:
+ # type: string
+ # default: ''
+ # steps:
+ # - run:
+ # name: Run dtests (<<parameters.file_tag>>)
+ # no_output_timeout: 15m
+ # command: |
+ # echo \"cat /tmp/split_dtest_tests_<<parameters.file_tag>>_final.txt\"
+ # cat /tmp/split_dtest_tests_<<parameters.file_tag>>_final.txt
+ #
+ # source ~/env/bin/activate
+ # export PATH=$JAVA_HOME/bin:$PATH
+ # if [ -n '<<parameters.extra_env_args>>' ]; then
+ # export <<parameters.extra_env_args>>
+ # fi
+ #
+ # java -version
+ # cd ~/cassandra-dtest
+ # mkdir -p /tmp/dtest
+ #
+ # echo \"env: $(env)\"
+ # echo \"** done env\"
+ # mkdir -p /tmp/results/dtests
+ # # we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ # export SPLIT_TESTS=`cat /tmp/split_dtest_tests_<<parameters.file_tag>>_final.txt`
+ # set -o pipefail && cd ~/cassandra-dtest && pytest <<parameters.pytest_extra_args>> --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_<<parameters.file_tag>>.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ # - store_test_results:
+ # path: /tmp/results
+ # - store_artifacts:
+ # path: /tmp/dtest
+ # destination: dtest_<<parameters.file_tag>>
+ # - store_artifacts:
+ # path: ~/cassandra-dtest/logs
+ # destination: dtest_<<parameters.file_tag>>_logs
diff --cc .circleci/config.yml.HIGHRES
index 0000000,ab38b45..de74d33
mode 000000,100644..100644
--- a/.circleci/config.yml.HIGHRES
+++ b/.circleci/config.yml.HIGHRES
@@@ -1,0 -1,1095 +1,1160 @@@
+ version: 2
+ jobs:
+ build:
+ docker:
+ - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Clone Cassandra Repository (via git)
+ command: |
+ git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH git://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra
+ - run:
+ name: Build Cassandra
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ cd ~/cassandra
+ # Loop to prevent failure due to maven-ant-tasks not downloading a jar..
+ for x in $(seq 1 3); do
+ ${ANT_HOME}/bin/ant clean jar
+ RETURN="$?"
+ if [ "${RETURN}" -eq "0" ]; then
+ break
+ fi
+ done
+ # Exit, if we didn't build successfully
+ if [ "${RETURN}" -ne "0" ]; then
+ echo "Build failed with exit code: ${RETURN}"
+ exit ${RETURN}
+ fi
+ no_output_timeout: 15m
+ - run:
+ name: Run eclipse-warnings
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ cd ~/cassandra
+ ant eclipse-warnings
+ - persist_to_workspace:
+ root: /home/cassandra
+ paths:
+ - cassandra
+ - .m2
+ environment:
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 2048M
+ - CCM_HEAP_NEWSIZE: 512M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_unit_tests:
++ j8_dtests-no-vnodes:
+ docker:
+ - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
- name: Determine Unit Tests to Run
- command: |
- # reminder: this code (along with all the steps) is independently executed on every circle container
- # so the goal here is to get the circleci script to return the tests *this* container will run
- # which we do via the `circleci` cli tool.
-
- rm -fr ~/cassandra-dtest/upgrade_tests
- echo "***java tests***"
-
- # get all of our unit test filenames
- set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
-
- # split up the unit tests into groups based on the number of containers we have
- set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
- set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | cut -c 37-1000000 | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
- echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
- cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
- no_output_timeout: 15m
- - run:
- name: Log Environment Information
++ name: Clone Cassandra dtest Repository (via git)
+ command: |
- echo '*** id ***'
- id
- echo '*** cat /proc/cpuinfo ***'
- cat /proc/cpuinfo
- echo '*** free -m ***'
- free -m
- echo '*** df -m ***'
- df -m
- echo '*** ifconfig -a ***'
- ifconfig -a
- echo '*** uname -a ***'
- uname -a
- echo '*** mount ***'
- mount
- echo '*** env ***'
- env
- echo '*** java ***'
- which java
- java -version
++ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
- name: Run Unit Tests (testclasslist)
++ name: Configure virtualenv and python Dependencies
+ command: |
++ # note, this should be super quick as all dependencies should be pre-installed in the docker image
++ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
++ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
++ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
++ source ~/env/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- ant testclasslist -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
++ pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
++ pip3 freeze
++ - run:
++ name: Determine Tests to Run (j8_without_vnodes)
++ no_output_timeout: 5m
++ command: "# reminder: this code (along with all the steps) is independently\
++ \ executed on every circle container\n# so the goal here is to get the circleci\
++ \ script to return the tests *this* container will run\n# which we do via\
++ \ the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\n\
++ export PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\n\
++ echo \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail &&\
++ \ ./run_dtests.py --skip-resource-intensive-tests --dtest-print-tests-only\
++ \ --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw\
++ \ --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw\
++ \ /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw\
++ \ > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match\
++ \ any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci\
++ \ tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes\
++ \ > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt\
++ \ | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\
++ cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n"
++ - run:
++ name: Run dtests (j8_without_vnodes)
+ no_output_timeout: 15m
++ command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\n\
++ cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env/bin/activate\n\
++ export PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\n\
++ java -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env:\
++ \ $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need\
++ \ the \"set -o pipefail\" here so that the exit code that circleci will\
++ \ actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat\
++ \ /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nset -o pipefail &&\
++ \ cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"\
++ INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml\
++ \ -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS\
++ \ 2>&1 | tee /tmp/dtest/stdout.txt\n"
+ - store_test_results:
- path: /tmp/cassandra/build/test/output/
++ path: /tmp/results
+ - store_artifacts:
- path: /tmp/cassandra/build/test/output
- destination: junitxml
++ path: /tmp/dtest
++ destination: dtest_j8_without_vnodes
+ - store_artifacts:
- path: /tmp/cassandra/build/test/logs
- destination: logs
++ path: ~/cassandra-dtest/logs
++ destination: dtest_j8_without_vnodes_logs
+ environment:
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 2048M
+ - CCM_HEAP_NEWSIZE: 512M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_jvm_dtests:
++ j8_upgradetests-no-vnodes:
+ docker:
+ - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
- parallelism: 1
++ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
- name: Run Unit Tests (test-jvm-dtest-forking)
++ name: Clone Cassandra dtest Repository (via git)
++ command: |
++ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
++ - run:
++ name: Configure virtualenv and python Dependencies
+ command: |
++ # note, this should be super quick as all dependencies should be pre-installed in the docker image
++ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
++ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
++ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
++ source ~/env/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- ant clean test-jvm-dtest-forking
++ pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
++ pip3 freeze
++ - run:
++ name: Determine Tests to Run (j8_upgradetests_without_vnodes)
++ no_output_timeout: 5m
++ command: |
++ # reminder: this code (along with all the steps) is independently executed on every circle container
++ # so the goal here is to get the circleci script to return the tests *this* container will run
++ # which we do via the `circleci` cli tool.
++
++ cd cassandra-dtest
++ source ~/env/bin/activate
++ export PATH=$JAVA_HOME/bin:$PATH
++
++ if [ -n 'RUN_STATIC_UPGRADE_MATRIX=true' ]; then
++ export RUN_STATIC_UPGRADE_MATRIX=true
++ fi
++
++ echo "***Collected DTests (j8_upgradetests_without_vnodes)***"
++ set -eo pipefail && ./run_dtests.py --execute-upgrade-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw --cassandra-dir=../cassandra
++ if [ -z '^upgrade_tests' ]; then
++ mv /tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw /tmp/all_dtest_tests_j8_upgradetests_without_vnodes
++ else
++ grep -e '^upgrade_tests' /tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw > /tmp/all_dtest_tests_j8_upgradetests_without_vnodes || { echo "Filter did not match any tests! Exiting build."; exit 0; }
++ fi
++ set -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_upgradetests_without_vnodes > /tmp/split_dtest_tests_j8_upgradetests_without_vnodes.txt
++ cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes.txt | tr '\n' ' ' > /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
++ cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
++ - run:
++ name: Run dtests (j8_upgradetests_without_vnodes)
+ no_output_timeout: 15m
++ command: |
++ echo "cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt"
++ cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
++
++ source ~/env/bin/activate
++ export PATH=$JAVA_HOME/bin:$PATH
++ if [ -n 'RUN_STATIC_UPGRADE_MATRIX=true' ]; then
++ export RUN_STATIC_UPGRADE_MATRIX=true
++ fi
++
++ java -version
++ cd ~/cassandra-dtest
++ mkdir -p /tmp/dtest
++
++ echo "env: $(env)"
++ echo "** done env"
++ mkdir -p /tmp/results/dtests
++ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
++ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt`
++ set -o pipefail && cd ~/cassandra-dtest && pytest --execute-upgrade-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_upgradetests_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
- path: /tmp/cassandra/build/test/output/
++ path: /tmp/results
+ - store_artifacts:
- path: /tmp/cassandra/build/test/output
- destination: junitxml
++ path: /tmp/dtest
++ destination: dtest_j8_upgradetests_without_vnodes
+ - store_artifacts:
- path: /tmp/cassandra/build/test/logs
- destination: logs
++ path: ~/cassandra-dtest/logs
++ destination: dtest_j8_upgradetests_without_vnodes_logs
+ environment:
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 2048M
+ - CCM_HEAP_NEWSIZE: 512M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- utests_long:
++ utests_stress:
+ docker:
+ - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
- name: Run Unit Tests (long-test)
++ name: Run Unit Tests (stress-test)
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
- ant clean long-test
++ ant clean stress-test
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 2048M
+ - CCM_HEAP_NEWSIZE: 512M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- utests_compression:
++ j8_unit_tests:
+ docker:
+ - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine Unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | cut -c 37-1000000 | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
- name: Run Unit Tests (testclasslist-compression)
++ name: Run Unit Tests (testclasslist)
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
- ant testclasslist-compression -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
++ ant testclasslist -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 2048M
+ - CCM_HEAP_NEWSIZE: 512M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_dtests-with-vnodes:
+ docker:
+ - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently\
+ \ executed on every circle container\n# so the goal here is to get the circleci\
+ \ script to return the tests *this* container will run\n# which we do via\
+ \ the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\n\
+ export PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\n\
+ echo \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py\
+ \ --use-vnodes --skip-resource-intensive-tests --dtest-print-tests-only\
+ \ --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\n\
+ if [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\n\
+ else\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes\
+ \ || { echo \"Filter did not match any tests! Exiting build.\"; exit 0;\
+ \ }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname\
+ \ /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\n\
+ cat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\
+ cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j8_with_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat\
+ \ /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env/bin/activate\n\
+ export PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\n\
+ java -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env:\
+ \ $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need\
+ \ the \"set -o pipefail\" here so that the exit code that circleci will\
+ \ actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat\
+ \ /tmp/split_dtest_tests_j8_with_vnodes_final.txt`\nset -o pipefail && cd\
+ \ ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests\
+ \ --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml\
+ \ -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS\
+ \ 2>&1 | tee /tmp/dtest/stdout.txt\n"
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j8_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j8_with_vnodes_logs
+ environment:
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 2048M
+ - CCM_HEAP_NEWSIZE: 512M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_dtests-no-vnodes:
++ j8_jvm_dtests:
+ docker:
+ - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
- parallelism: 100
++ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
- name: Clone Cassandra dtest Repository (via git)
- command: |
- git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- - run:
- name: Configure virtualenv and python Dependencies
++ name: Run Unit Tests (test-jvm-dtest-forking)
+ command: |
- # note, this should be super quick as all dependencies should be pre-installed in the docker image
- # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
- # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
- # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
- source ~/env/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
- pip3 freeze
- - run:
- name: Determine Tests to Run (j8_without_vnodes)
- no_output_timeout: 5m
- command: "# reminder: this code (along with all the steps) is independently\
- \ executed on every circle container\n# so the goal here is to get the circleci\
- \ script to return the tests *this* container will run\n# which we do via\
- \ the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\n\
- export PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\n\
- echo \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail &&\
- \ ./run_dtests.py --skip-resource-intensive-tests --dtest-print-tests-only\
- \ --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw\
- \ --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw\
- \ /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw\
- \ > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match\
- \ any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci\
- \ tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes\
- \ > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt\
- \ | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\
- cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n"
- - run:
- name: Run dtests (j8_without_vnodes)
++ time mv ~/cassandra /tmp
++ cd /tmp/cassandra
++ ant clean test-jvm-dtest-forking
+ no_output_timeout: 15m
- command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\n\
- cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env/bin/activate\n\
- export PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\n\
- java -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env:\
- \ $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need\
- \ the \"set -o pipefail\" here so that the exit code that circleci will\
- \ actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat\
- \ /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nset -o pipefail &&\
- \ cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"\
- INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml\
- \ -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS\
- \ 2>&1 | tee /tmp/dtest/stdout.txt\n"
+ - store_test_results:
- path: /tmp/results
++ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
- path: /tmp/dtest
- destination: dtest_j8_without_vnodes
++ path: /tmp/cassandra/build/test/output
++ destination: junitxml
+ - store_artifacts:
- path: ~/cassandra-dtest/logs
- destination: dtest_j8_without_vnodes_logs
++ path: /tmp/cassandra/build/test/logs
++ destination: logs
+ environment:
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 2048M
+ - CCM_HEAP_NEWSIZE: 512M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_upgradetests-no-vnodes:
++ utests_long:
+ docker:
+ - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
- parallelism: 100
++ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
- name: Clone Cassandra dtest Repository (via git)
- command: |
- git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- - run:
- name: Configure virtualenv and python Dependencies
++ name: Run Unit Tests (long-test)
+ command: |
- # note, this should be super quick as all dependencies should be pre-installed in the docker image
- # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
- # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
- # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
- source ~/env/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
- pip3 freeze
++ time mv ~/cassandra /tmp
++ cd /tmp/cassandra
++ ant clean long-test
++ no_output_timeout: 15m
++ - store_test_results:
++ path: /tmp/cassandra/build/test/output/
++ - store_artifacts:
++ path: /tmp/cassandra/build/test/output
++ destination: junitxml
++ - store_artifacts:
++ path: /tmp/cassandra/build/test/logs
++ destination: logs
++ environment:
++ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ - ANT_HOME: /usr/share/ant
++ - LANG: en_US.UTF-8
++ - KEEP_TEST_DIR: true
++ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
++ - PYTHONIOENCODING: utf-8
++ - PYTHONUNBUFFERED: true
++ - CASS_DRIVER_NO_EXTENSIONS: true
++ - CASS_DRIVER_NO_CYTHON: true
++ - CASSANDRA_SKIP_SYNC: true
++ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
++ - DTEST_BRANCH: master
++ - CCM_MAX_HEAP_SIZE: 2048M
++ - CCM_HEAP_NEWSIZE: 512M
++ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ utests_compression:
++ docker:
++ - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
++ resource_class: xlarge
++ working_directory: ~/
++ shell: /bin/bash -eo pipefail -l
++ parallelism: 100
++ steps:
++ - attach_workspace:
++ at: /home/cassandra
+ - run:
- name: Determine Tests to Run (j8_upgradetests_without_vnodes)
- no_output_timeout: 5m
++ name: Determine Unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
- cd cassandra-dtest
- source ~/env/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
++ rm -fr ~/cassandra-dtest/upgrade_tests
++ echo "***java tests***"
+
- if [ -n 'RUN_STATIC_UPGRADE_MATRIX=true' ]; then
- export RUN_STATIC_UPGRADE_MATRIX=true
- fi
++ # get all of our unit test filenames
++ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
- echo "***Collected DTests (j8_upgradetests_without_vnodes)***"
- set -eo pipefail && ./run_dtests.py --execute-upgrade-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw --cassandra-dir=../cassandra
- if [ -z '^upgrade_tests' ]; then
- mv /tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw /tmp/all_dtest_tests_j8_upgradetests_without_vnodes
- else
- grep -e '^upgrade_tests' /tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw > /tmp/all_dtest_tests_j8_upgradetests_without_vnodes || { echo "Filter did not match any tests! Exiting build."; exit 0; }
- fi
- set -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_upgradetests_without_vnodes > /tmp/split_dtest_tests_j8_upgradetests_without_vnodes.txt
- cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes.txt | tr '\n' ' ' > /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
- cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
- - run:
- name: Run dtests (j8_upgradetests_without_vnodes)
++ # split up the unit tests into groups based on the number of containers we have
++ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
++ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | cut -c 37-1000000 | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
++ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
++ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
++ - run:
++ name: Log Environment Information
+ command: |
- echo "cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt"
- cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
-
- source ~/env/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
- if [ -n 'RUN_STATIC_UPGRADE_MATRIX=true' ]; then
- export RUN_STATIC_UPGRADE_MATRIX=true
- fi
-
++ echo '*** id ***'
++ id
++ echo '*** cat /proc/cpuinfo ***'
++ cat /proc/cpuinfo
++ echo '*** free -m ***'
++ free -m
++ echo '*** df -m ***'
++ df -m
++ echo '*** ifconfig -a ***'
++ ifconfig -a
++ echo '*** uname -a ***'
++ uname -a
++ echo '*** mount ***'
++ mount
++ echo '*** env ***'
++ env
++ echo '*** java ***'
++ which java
+ java -version
- cd ~/cassandra-dtest
- mkdir -p /tmp/dtest
-
- echo "env: $(env)"
- echo "** done env"
- mkdir -p /tmp/results/dtests
- # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
- export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt`
- set -o pipefail && cd ~/cassandra-dtest && pytest --execute-upgrade-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_upgradetests_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
++ - run:
++ name: Run Unit Tests (testclasslist-compression)
++ command: |
++ export PATH=$JAVA_HOME/bin:$PATH
++ time mv ~/cassandra /tmp
++ cd /tmp/cassandra
++ ant testclasslist-compression -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
++ no_output_timeout: 15m
+ - store_test_results:
- path: /tmp/results
++ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
- path: /tmp/dtest
- destination: dtest_j8_upgradetests_without_vnodes
++ path: /tmp/cassandra/build/test/output
++ destination: junitxml
+ - store_artifacts:
- path: ~/cassandra-dtest/logs
- destination: dtest_j8_upgradetests_without_vnodes_logs
++ path: /tmp/cassandra/build/test/logs
++ destination: logs
+ environment:
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 2048M
+ - CCM_HEAP_NEWSIZE: 512M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ workflows:
+ version: 2
+ build_and_run_tests:
+ jobs:
+ - build
+ - j8_unit_tests:
+ requires:
+ - build
+ - j8_jvm_dtests:
+ requires:
+ - build
+ - start_utests_long:
+ type: approval
+ requires:
+ - build
+ - utests_long:
+ requires:
+ - start_utests_long
+ - start_utests_compression:
+ type: approval
+ requires:
+ - build
+ - utests_compression:
+ requires:
+ - start_utests_compression
++ - start_utests_stress:
++ type: approval
++ requires:
++ - build
++ - utests_stress:
++ requires:
++ - start_utests_stress
+ - start_j8_dtests:
+ type: approval
+ requires:
+ - build
+ - j8_dtests-with-vnodes:
+ requires:
+ - start_j8_dtests
+ - j8_dtests-no-vnodes:
+ requires:
+ - start_j8_dtests
+ - start_upgrade_tests:
+ type: approval
+ requires:
+ - build
+ - j8_upgradetests-no-vnodes:
+ requires:
+ - start_upgrade_tests
+
+ # Original config.yml file:
+ # version: 2.1
+ #
+ # default_env_vars: &default_env_vars
+ # JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ # ANT_HOME: /usr/share/ant
+ # LANG: en_US.UTF-8
+ # KEEP_TEST_DIR: true
+ # DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ # PYTHONIOENCODING: utf-8
+ # PYTHONUNBUFFERED: true
+ # CASS_DRIVER_NO_EXTENSIONS: true
+ # CASS_DRIVER_NO_CYTHON: true
+ # #Skip all syncing to disk to avoid performance issues in flaky CI environments
+ # CASSANDRA_SKIP_SYNC: true
+ # DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ # DTEST_BRANCH: master
+ # CCM_MAX_HEAP_SIZE: 2048M
+ # CCM_HEAP_NEWSIZE: 512M
+ #
+ # j8_par_executor: &j8_par_executor
+ # executor:
+ # name: java8-executor
+ # exec_resource_class: xlarge
+ # parallelism: 100
+ #
+ # j8_seq_executor: &j8_seq_executor
+ # executor:
+ # name: java8-executor
+ # exec_resource_class: xlarge
+ # parallelism: 1 # sequential, single container tests: no parallelism benefits
+ #
+ # with_dtests_jobs: &with_dtest_jobs
+ # jobs:
+ # - build
+ # # Java 8 unit tests will be run automatically
+ # - j8_unit_tests:
+ # requires:
+ # - build
+ # - j8_jvm_dtests:
+ # requires:
+ # - build
+ # # specialized unit tests (all run on request using Java 8)
+ # - start_utests_long:
+ # type: approval
+ # requires:
+ # - build
+ # - utests_long:
+ # requires:
+ # - start_utests_long
+ # - start_utests_compression:
+ # type: approval
+ # requires:
+ # - build
+ # - utests_compression:
+ # requires:
+ # - start_utests_compression
++# - start_utests_stress:
++# type: approval
++# requires:
++# - build
++# - utests_stress:
++# requires:
++# - start_utests_stress
+ # # Java 8 dtests (on request)
+ # - start_j8_dtests:
+ # type: approval
+ # requires:
+ # - build
+ # - j8_dtests-with-vnodes:
+ # requires:
+ # - start_j8_dtests
+ # - j8_dtests-no-vnodes:
+ # requires:
+ # - start_j8_dtests
+ # # Java 8 upgrade tests
+ # - start_upgrade_tests:
+ # type: approval
+ # requires:
+ # - build
+ # - j8_upgradetests-no-vnodes:
+ # requires:
+ # - start_upgrade_tests
+ #
+ # with_dtest_jobs_only: &with_dtest_jobs_only
+ # jobs:
+ # - build
+ # - j8_dtests-with-vnodes:
+ # requires:
+ # - build
+ # - j8_dtests-no-vnodes:
+ # requires:
+ # - build
+ #
+ # workflows:
+ # version: 2
+ # build_and_run_tests: *with_dtest_jobs
+ # #build_and_run_tests: *with_dtest_jobs_only
+ #
+ # executors:
+ # java8-executor:
+ # parameters:
+ # exec_resource_class:
+ # type: string
+ # default: medium
+ # docker:
+ # - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ # resource_class: << parameters.exec_resource_class >>
+ # working_directory: ~/
+ # shell: /bin/bash -eo pipefail -l
+ # environment:
+ # <<: *default_env_vars
+ # JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ # JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ #
+ # jobs:
+ # build:
+ # executor: java8-executor
+ # parallelism: 1 # This job doesn't benefit from parallelism
+ # steps:
+ # - log_environment
+ # - clone_cassandra
+ # - build_cassandra
+ # - run_eclipse_warnings
+ # - persist_to_workspace:
+ # root: /home/cassandra
+ # paths:
+ # - cassandra
+ # - .m2
+ #
+ # j8_unit_tests:
+ # <<: *j8_par_executor
+ # steps:
+ # - attach_workspace:
+ # at: /home/cassandra
+ # - create_junit_containers
+ # - log_environment
+ # - run_parallel_junit_tests
+ #
+ # j8_jvm_dtests:
+ # <<: *j8_seq_executor
+ # steps:
+ # - attach_workspace:
+ # at: /home/cassandra
+ # - run_junit_tests:
+ # target: test-jvm-dtest-forking
+ #
+ # utests_long:
+ # <<: *j8_seq_executor
+ # steps:
+ # - attach_workspace:
+ # at: /home/cassandra
+ # - run_junit_tests:
+ # target: long-test
+ #
+ # utests_compression:
+ # <<: *j8_par_executor
+ # steps:
+ # - attach_workspace:
+ # at: /home/cassandra
+ # - create_junit_containers
+ # - log_environment
+ # - run_parallel_junit_tests:
+ # target: testclasslist-compression
+ #
++# utests_stress:
++# <<: *j8_seq_executor
++# steps:
++# - attach_workspace:
++# at: /home/cassandra
++# - run_junit_tests:
++# target: stress-test
++#
+ # j8_dtests-with-vnodes:
+ # <<: *j8_par_executor
+ # steps:
+ # - attach_workspace:
+ # at: /home/cassandra
+ # - clone_dtest
+ # - create_venv
+ # - create_dtest_containers:
+ # file_tag: j8_with_vnodes
+ # run_dtests_extra_args: '--use-vnodes --skip-resource-intensive-tests'
+ # - run_dtests:
+ # file_tag: j8_with_vnodes
+ # pytest_extra_args: '--use-vnodes --num-tokens=32 --skip-resource-intensive-tests'
+ #
+ # j8_dtests-no-vnodes:
+ # <<: *j8_par_executor
+ # steps:
+ # - attach_workspace:
+ # at: /home/cassandra
+ # - clone_dtest
+ # - create_venv
+ # - create_dtest_containers:
+ # file_tag: j8_without_vnodes
+ # run_dtests_extra_args: '--skip-resource-intensive-tests'
+ # - run_dtests:
+ # file_tag: j8_without_vnodes
+ # pytest_extra_args: '--skip-resource-intensive-tests'
+ #
+ # j8_upgradetests-no-vnodes:
+ # <<: *j8_par_executor
+ # steps:
+ # - attach_workspace:
+ # at: /home/cassandra
+ # - clone_dtest
+ # - create_venv
+ # - create_dtest_containers:
+ # file_tag: j8_upgradetests_without_vnodes
+ # run_dtests_extra_args: '--execute-upgrade-tests'
+ # extra_env_args: 'RUN_STATIC_UPGRADE_MATRIX=true'
+ # tests_filter_pattern: '^upgrade_tests'
+ # - run_dtests:
+ # file_tag: j8_upgradetests_without_vnodes
+ # extra_env_args: 'RUN_STATIC_UPGRADE_MATRIX=true'
+ # pytest_extra_args: '--execute-upgrade-tests'
+ #
+ # commands:
+ # log_environment:
+ # steps:
+ # - run:
+ # name: Log Environment Information
+ # command: |
+ # echo '*** id ***'
+ # id
+ # echo '*** cat /proc/cpuinfo ***'
+ # cat /proc/cpuinfo
+ # echo '*** free -m ***'
+ # free -m
+ # echo '*** df -m ***'
+ # df -m
+ # echo '*** ifconfig -a ***'
+ # ifconfig -a
+ # echo '*** uname -a ***'
+ # uname -a
+ # echo '*** mount ***'
+ # mount
+ # echo '*** env ***'
+ # env
+ # echo '*** java ***'
+ # which java
+ # java -version
+ #
+ # clone_cassandra:
+ # steps:
+ # - run:
+ # name: Clone Cassandra Repository (via git)
+ # command: |
+ # git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH git://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra
+ #
+ # clone_dtest:
+ # steps:
+ # - run:
+ # name: Clone Cassandra dtest Repository (via git)
+ # command: |
+ # git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ #
+ # build_cassandra:
+ # steps:
+ # - run:
+ # name: Build Cassandra
+ # command: |
+ # export PATH=$JAVA_HOME/bin:$PATH
+ # cd ~/cassandra
+ # # Loop to prevent failure due to maven-ant-tasks not downloading a jar..
+ # for x in $(seq 1 3); do
+ # ${ANT_HOME}/bin/ant clean jar
+ # RETURN=\"$?\"
+ # if [ \"${RETURN}\" -eq \"0\" ]; then
+ # break
+ # fi
+ # done
+ # # Exit, if we didn't build successfully
+ # if [ \"${RETURN}\" -ne \"0\" ]; then
+ # echo \"Build failed with exit code: ${RETURN}\"
+ # exit ${RETURN}
+ # fi
+ # no_output_timeout: 15m
+ #
+ # run_eclipse_warnings:
+ # steps:
+ # - run:
+ # name: Run eclipse-warnings
+ # command: |
+ # export PATH=$JAVA_HOME/bin:$PATH
+ # cd ~/cassandra
+ # ant eclipse-warnings
+ #
+ # create_junit_containers:
+ # steps:
+ # - run:
+ # name: Determine Unit Tests to Run
+ # command: |
+ # # reminder: this code (along with all the steps) is independently executed on every circle container
+ # # so the goal here is to get the circleci script to return the tests *this* container will run
+ # # which we do via the `circleci` cli tool.
+ #
+ # rm -fr ~/cassandra-dtest/upgrade_tests
+ # echo \"***java tests***\"
+ #
+ # # get all of our unit test filenames
+ # set -eo pipefail && circleci tests glob \"$HOME/cassandra/test/unit/**/*.java\" > /tmp/all_java_unit_tests.txt
+ #
+ # # split up the unit tests into groups based on the number of containers we have
+ # set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ # set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | cut -c 37-1000000 | grep \"Test\\.java$\" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ # echo \"** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt\"
+ # cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ #
+ # no_output_timeout: 15m
+ #
+ # run_junit_tests:
+ # parameters:
+ # target:
+ # type: string
+ # no_output_timeout:
+ # type: string
+ # default: 15m
+ # steps:
+ # - run:
+ # name: Run Unit Tests (<<parameters.target>>)
+ # # Please note that we run `clean` and therefore rebuild the project, as we can't run tests on Java 8 in case
+ # # based on Java 11 builds.
+ # command: |
+ # export PATH=$JAVA_HOME/bin:$PATH
+ # time mv ~/cassandra /tmp
+ # cd /tmp/cassandra
+ # ant clean <<parameters.target>>
+ # no_output_timeout: <<parameters.no_output_timeout>>
+ # - store_test_results:
+ # path: /tmp/cassandra/build/test/output/
+ # - store_artifacts:
+ # path: /tmp/cassandra/build/test/output
+ # destination: junitxml
+ # - store_artifacts:
+ # path: /tmp/cassandra/build/test/logs
+ # destination: logs
+ #
+ # run_parallel_junit_tests:
+ # parameters:
+ # target:
+ # type: string
+ # default: testclasslist
+ # no_output_timeout:
+ # type: string
+ # default: 15m
+ # steps:
+ # - run:
+ # name: Run Unit Tests (<<parameters.target>>)
+ # # Please note that we run `clean` and therefore rebuild the project, as we can't run tests on Java 8 in case
+ # # based on Java 11 builds.
+ # command: |
+ # export PATH=$JAVA_HOME/bin:$PATH
+ # time mv ~/cassandra /tmp
+ # cd /tmp/cassandra
+ # ant <<parameters.target>> -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
+ # no_output_timeout: <<parameters.no_output_timeout>>
+ # - store_test_results:
+ # path: /tmp/cassandra/build/test/output/
+ # - store_artifacts:
+ # path: /tmp/cassandra/build/test/output
+ # destination: junitxml
+ # - store_artifacts:
+ # path: /tmp/cassandra/build/test/logs
+ # destination: logs
+ #
+ # create_venv:
+ # steps:
+ # - run:
+ # name: Configure virtualenv and python Dependencies
+ # command: |
+ # # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ # source ~/env/bin/activate
+ # export PATH=$JAVA_HOME/bin:$PATH
+ # pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
+ # pip3 freeze
+ #
+ # create_dtest_containers:
+ # parameters:
+ # file_tag:
+ # type: string
+ # run_dtests_extra_args:
+ # type: string
+ # default: ''
+ # extra_env_args:
+ # type: string
+ # default: ''
+ # tests_filter_pattern:
+ # type: string
+ # default: ''
+ # steps:
+ # - run:
+ # name: Determine Tests to Run (<<parameters.file_tag>>)
+ # no_output_timeout: 5m
+ # command: |
+ # # reminder: this code (along with all the steps) is independently executed on every circle container
+ # # so the goal here is to get the circleci script to return the tests *this* container will run
+ # # which we do via the `circleci` cli tool.
+ #
+ # cd cassandra-dtest
+ # source ~/env/bin/activate
+ # export PATH=$JAVA_HOME/bin:$PATH
+ #
+ # if [ -n '<<parameters.extra_env_args>>' ]; then
+ # export <<parameters.extra_env_args>>
+ # fi
+ #
+ # echo \"***Collected DTests (<<parameters.file_tag>>)***\"
+ # set -eo pipefail && ./run_dtests.py <<parameters.run_dtests_extra_args>> --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_<<parameters.file_tag>>_raw --cassandra-dir=../cassandra
+ # if [ -z '<<parameters.tests_filter_pattern>>' ]; then
+ # mv /tmp/all_dtest_tests_<<parameters.file_tag>>_raw /tmp/all_dtest_tests_<<parameters.file_tag>>
+ # else
+ # grep -e '<<parameters.tests_filter_pattern>>' /tmp/all_dtest_tests_<<parameters.file_tag>>_raw > /tmp/all_dtest_tests_<<parameters.file_tag>> || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }
+ # fi
+ # set -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_<<parameters.file_tag>> > /tmp/split_dtest_tests_<<parameters.file_tag>>.txt
+ # cat /tmp/split_dtest_tests_<<parameters.file_tag>>.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_<<parameters.file_tag>>_final.txt
+ # cat /tmp/split_dtest_tests_<<parameters.file_tag>>_final.txt
+ #
+ # run_dtests:
+ # parameters:
+ # file_tag:
+ # type: string
+ # pytest_extra_args:
+ # type: string
+ # default: ''
+ # extra_env_args:
+ # type: string
+ # default: ''
+ # steps:
+ # - run:
+ # name: Run dtests (<<parameters.file_tag>>)
+ # no_output_timeout: 15m
+ # command: |
+ # echo \"cat /tmp/split_dtest_tests_<<parameters.file_tag>>_final.txt\"
+ # cat /tmp/split_dtest_tests_<<parameters.file_tag>>_final.txt
+ #
+ # source ~/env/bin/activate
+ # export PATH=$JAVA_HOME/bin:$PATH
+ # if [ -n '<<parameters.extra_env_args>>' ]; then
+ # export <<parameters.extra_env_args>>
+ # fi
+ #
+ # java -version
+ # cd ~/cassandra-dtest
+ # mkdir -p /tmp/dtest
+ #
+ # echo \"env: $(env)\"
+ # echo \"** done env\"
+ # mkdir -p /tmp/results/dtests
+ # # we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ # export SPLIT_TESTS=`cat /tmp/split_dtest_tests_<<parameters.file_tag>>_final.txt`
+ # set -o pipefail && cd ~/cassandra-dtest && pytest <<parameters.pytest_extra_args>> --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_<<parameters.file_tag>>.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ # - store_test_results:
+ # path: /tmp/results
+ # - store_artifacts:
+ # path: /tmp/dtest
+ # destination: dtest_<<parameters.file_tag>>
+ # - store_artifacts:
+ # path: ~/cassandra-dtest/logs
+ # destination: dtest_<<parameters.file_tag>>_logs
diff --cc .circleci/config.yml.LOWRES
index 0000000,0d56e23..7016d9b
mode 000000,100644..100644
--- a/.circleci/config.yml.LOWRES
+++ b/.circleci/config.yml.LOWRES
@@@ -1,0 -1,1095 +1,1160 @@@
+ version: 2
+ jobs:
+ build:
+ docker:
+ - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Clone Cassandra Repository (via git)
+ command: |
+ git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH git://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra
+ - run:
+ name: Build Cassandra
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ cd ~/cassandra
+ # Loop to prevent failure due to maven-ant-tasks not downloading a jar..
+ for x in $(seq 1 3); do
+ ${ANT_HOME}/bin/ant clean jar
+ RETURN="$?"
+ if [ "${RETURN}" -eq "0" ]; then
+ break
+ fi
+ done
+ # Exit, if we didn't build successfully
+ if [ "${RETURN}" -ne "0" ]; then
+ echo "Build failed with exit code: ${RETURN}"
+ exit ${RETURN}
+ fi
+ no_output_timeout: 15m
+ - run:
+ name: Run eclipse-warnings
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ cd ~/cassandra
+ ant eclipse-warnings
+ - persist_to_workspace:
+ root: /home/cassandra
+ paths:
+ - cassandra
+ - .m2
+ environment:
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_unit_tests:
++ j8_dtests-no-vnodes:
+ docker:
+ - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
- name: Determine Unit Tests to Run
- command: |
- # reminder: this code (along with all the steps) is independently executed on every circle container
- # so the goal here is to get the circleci script to return the tests *this* container will run
- # which we do via the `circleci` cli tool.
-
- rm -fr ~/cassandra-dtest/upgrade_tests
- echo "***java tests***"
-
- # get all of our unit test filenames
- set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
-
- # split up the unit tests into groups based on the number of containers we have
- set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
- set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | cut -c 37-1000000 | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
- echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
- cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
- no_output_timeout: 15m
- - run:
- name: Log Environment Information
++ name: Clone Cassandra dtest Repository (via git)
+ command: |
- echo '*** id ***'
- id
- echo '*** cat /proc/cpuinfo ***'
- cat /proc/cpuinfo
- echo '*** free -m ***'
- free -m
- echo '*** df -m ***'
- df -m
- echo '*** ifconfig -a ***'
- ifconfig -a
- echo '*** uname -a ***'
- uname -a
- echo '*** mount ***'
- mount
- echo '*** env ***'
- env
- echo '*** java ***'
- which java
- java -version
++ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
- name: Run Unit Tests (testclasslist)
++ name: Configure virtualenv and python Dependencies
+ command: |
++ # note, this should be super quick as all dependencies should be pre-installed in the docker image
++ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
++ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
++ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
++ source ~/env/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- ant testclasslist -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
++ pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
++ pip3 freeze
++ - run:
++ name: Determine Tests to Run (j8_without_vnodes)
++ no_output_timeout: 5m
++ command: "# reminder: this code (along with all the steps) is independently\
++ \ executed on every circle container\n# so the goal here is to get the circleci\
++ \ script to return the tests *this* container will run\n# which we do via\
++ \ the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\n\
++ export PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\n\
++ echo \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail &&\
++ \ ./run_dtests.py --skip-resource-intensive-tests --dtest-print-tests-only\
++ \ --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw\
++ \ --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw\
++ \ /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw\
++ \ > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match\
++ \ any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci\
++ \ tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes\
++ \ > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt\
++ \ | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\
++ cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n"
++ - run:
++ name: Run dtests (j8_without_vnodes)
+ no_output_timeout: 15m
++ command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\n\
++ cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env/bin/activate\n\
++ export PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\n\
++ java -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env:\
++ \ $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need\
++ \ the \"set -o pipefail\" here so that the exit code that circleci will\
++ \ actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat\
++ \ /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nset -o pipefail &&\
++ \ cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"\
++ INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml\
++ \ -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS\
++ \ 2>&1 | tee /tmp/dtest/stdout.txt\n"
+ - store_test_results:
- path: /tmp/cassandra/build/test/output/
++ path: /tmp/results
+ - store_artifacts:
- path: /tmp/cassandra/build/test/output
- destination: junitxml
++ path: /tmp/dtest
++ destination: dtest_j8_without_vnodes
+ - store_artifacts:
- path: /tmp/cassandra/build/test/logs
- destination: logs
++ path: ~/cassandra-dtest/logs
++ destination: dtest_j8_without_vnodes_logs
+ environment:
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_jvm_dtests:
++ j8_upgradetests-no-vnodes:
+ docker:
+ - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
- parallelism: 1
++ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
- name: Run Unit Tests (test-jvm-dtest-forking)
++ name: Clone Cassandra dtest Repository (via git)
++ command: |
++ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
++ - run:
++ name: Configure virtualenv and python Dependencies
+ command: |
++ # note, this should be super quick as all dependencies should be pre-installed in the docker image
++ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
++ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
++ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
++ source ~/env/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- ant clean test-jvm-dtest-forking
++ pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
++ pip3 freeze
++ - run:
++ name: Determine Tests to Run (j8_upgradetests_without_vnodes)
++ no_output_timeout: 5m
++ command: |
++ # reminder: this code (along with all the steps) is independently executed on every circle container
++ # so the goal here is to get the circleci script to return the tests *this* container will run
++ # which we do via the `circleci` cli tool.
++
++ cd cassandra-dtest
++ source ~/env/bin/activate
++ export PATH=$JAVA_HOME/bin:$PATH
++
++ if [ -n 'RUN_STATIC_UPGRADE_MATRIX=true' ]; then
++ export RUN_STATIC_UPGRADE_MATRIX=true
++ fi
++
++ echo "***Collected DTests (j8_upgradetests_without_vnodes)***"
++ set -eo pipefail && ./run_dtests.py --execute-upgrade-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw --cassandra-dir=../cassandra
++ if [ -z '^upgrade_tests' ]; then
++ mv /tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw /tmp/all_dtest_tests_j8_upgradetests_without_vnodes
++ else
++ grep -e '^upgrade_tests' /tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw > /tmp/all_dtest_tests_j8_upgradetests_without_vnodes || { echo "Filter did not match any tests! Exiting build."; exit 0; }
++ fi
++ set -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_upgradetests_without_vnodes > /tmp/split_dtest_tests_j8_upgradetests_without_vnodes.txt
++ cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes.txt | tr '\n' ' ' > /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
++ cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
++ - run:
++ name: Run dtests (j8_upgradetests_without_vnodes)
+ no_output_timeout: 15m
++ command: |
++ echo "cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt"
++ cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
++
++ source ~/env/bin/activate
++ export PATH=$JAVA_HOME/bin:$PATH
++ if [ -n 'RUN_STATIC_UPGRADE_MATRIX=true' ]; then
++ export RUN_STATIC_UPGRADE_MATRIX=true
++ fi
++
++ java -version
++ cd ~/cassandra-dtest
++ mkdir -p /tmp/dtest
++
++ echo "env: $(env)"
++ echo "** done env"
++ mkdir -p /tmp/results/dtests
++ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
++ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt`
++ set -o pipefail && cd ~/cassandra-dtest && pytest --execute-upgrade-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_upgradetests_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
- path: /tmp/cassandra/build/test/output/
++ path: /tmp/results
+ - store_artifacts:
- path: /tmp/cassandra/build/test/output
- destination: junitxml
++ path: /tmp/dtest
++ destination: dtest_j8_upgradetests_without_vnodes
+ - store_artifacts:
- path: /tmp/cassandra/build/test/logs
- destination: logs
++ path: ~/cassandra-dtest/logs
++ destination: dtest_j8_upgradetests_without_vnodes_logs
+ environment:
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- utests_long:
++ utests_stress:
+ docker:
+ - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
- name: Run Unit Tests (long-test)
++ name: Run Unit Tests (stress-test)
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
- ant clean long-test
++ ant clean stress-test
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- utests_compression:
++ j8_unit_tests:
+ docker:
+ - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine Unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | cut -c 37-1000000 | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
- name: Run Unit Tests (testclasslist-compression)
++ name: Run Unit Tests (testclasslist)
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
- ant testclasslist-compression -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
++ ant testclasslist -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_dtests-with-vnodes:
+ docker:
+ - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently\
+ \ executed on every circle container\n# so the goal here is to get the circleci\
+ \ script to return the tests *this* container will run\n# which we do via\
+ \ the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\n\
+ export PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\n\
+ echo \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py\
+ \ --use-vnodes --skip-resource-intensive-tests --dtest-print-tests-only\
+ \ --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\n\
+ if [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\n\
+ else\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes\
+ \ || { echo \"Filter did not match any tests! Exiting build.\"; exit 0;\
+ \ }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname\
+ \ /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\n\
+ cat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\
+ cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n"
+ - run:
+ name: Run dtests (j8_with_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat\
+ \ /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env/bin/activate\n\
+ export PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\n\
+ java -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env:\
+ \ $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need\
+ \ the \"set -o pipefail\" here so that the exit code that circleci will\
+ \ actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat\
+ \ /tmp/split_dtest_tests_j8_with_vnodes_final.txt`\nset -o pipefail && cd\
+ \ ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests\
+ \ --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml\
+ \ -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS\
+ \ 2>&1 | tee /tmp/dtest/stdout.txt\n"
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j8_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j8_with_vnodes_logs
+ environment:
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_dtests-no-vnodes:
++ j8_jvm_dtests:
+ docker:
+ - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
- parallelism: 4
++ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
- name: Clone Cassandra dtest Repository (via git)
- command: |
- git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- - run:
- name: Configure virtualenv and python Dependencies
++ name: Run Unit Tests (test-jvm-dtest-forking)
+ command: |
- # note, this should be super quick as all dependencies should be pre-installed in the docker image
- # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
- # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
- # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
- source ~/env/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
- pip3 freeze
- - run:
- name: Determine Tests to Run (j8_without_vnodes)
- no_output_timeout: 5m
- command: "# reminder: this code (along with all the steps) is independently\
- \ executed on every circle container\n# so the goal here is to get the circleci\
- \ script to return the tests *this* container will run\n# which we do via\
- \ the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\n\
- export PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\n\
- echo \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail &&\
- \ ./run_dtests.py --skip-resource-intensive-tests --dtest-print-tests-only\
- \ --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw\
- \ --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw\
- \ /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw\
- \ > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match\
- \ any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci\
- \ tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes\
- \ > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt\
- \ | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\
- cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n"
- - run:
- name: Run dtests (j8_without_vnodes)
++ time mv ~/cassandra /tmp
++ cd /tmp/cassandra
++ ant clean test-jvm-dtest-forking
+ no_output_timeout: 15m
- command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\n\
- cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env/bin/activate\n\
- export PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\n\
- java -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env:\
- \ $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need\
- \ the \"set -o pipefail\" here so that the exit code that circleci will\
- \ actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat\
- \ /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nset -o pipefail &&\
- \ cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"\
- INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml\
- \ -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS\
- \ 2>&1 | tee /tmp/dtest/stdout.txt\n"
+ - store_test_results:
- path: /tmp/results
++ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
- path: /tmp/dtest
- destination: dtest_j8_without_vnodes
++ path: /tmp/cassandra/build/test/output
++ destination: junitxml
+ - store_artifacts:
- path: ~/cassandra-dtest/logs
- destination: dtest_j8_without_vnodes_logs
++ path: /tmp/cassandra/build/test/logs
++ destination: logs
+ environment:
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_upgradetests-no-vnodes:
++ utests_long:
+ docker:
+ - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
- parallelism: 4
++ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
- name: Clone Cassandra dtest Repository (via git)
- command: |
- git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- - run:
- name: Configure virtualenv and python Dependencies
++ name: Run Unit Tests (long-test)
+ command: |
- # note, this should be super quick as all dependencies should be pre-installed in the docker image
- # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
- # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
- # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
- source ~/env/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
- pip3 freeze
++ time mv ~/cassandra /tmp
++ cd /tmp/cassandra
++ ant clean long-test
++ no_output_timeout: 15m
++ - store_test_results:
++ path: /tmp/cassandra/build/test/output/
++ - store_artifacts:
++ path: /tmp/cassandra/build/test/output
++ destination: junitxml
++ - store_artifacts:
++ path: /tmp/cassandra/build/test/logs
++ destination: logs
++ environment:
++ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ - ANT_HOME: /usr/share/ant
++ - LANG: en_US.UTF-8
++ - KEEP_TEST_DIR: true
++ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
++ - PYTHONIOENCODING: utf-8
++ - PYTHONUNBUFFERED: true
++ - CASS_DRIVER_NO_EXTENSIONS: true
++ - CASS_DRIVER_NO_CYTHON: true
++ - CASSANDRA_SKIP_SYNC: true
++ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
++ - DTEST_BRANCH: master
++ - CCM_MAX_HEAP_SIZE: 1024M
++ - CCM_HEAP_NEWSIZE: 256M
++ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ utests_compression:
++ docker:
++ - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
++ resource_class: medium
++ working_directory: ~/
++ shell: /bin/bash -eo pipefail -l
++ parallelism: 4
++ steps:
++ - attach_workspace:
++ at: /home/cassandra
+ - run:
- name: Determine Tests to Run (j8_upgradetests_without_vnodes)
- no_output_timeout: 5m
++ name: Determine Unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
- cd cassandra-dtest
- source ~/env/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
++ rm -fr ~/cassandra-dtest/upgrade_tests
++ echo "***java tests***"
+
- if [ -n 'RUN_STATIC_UPGRADE_MATRIX=true' ]; then
- export RUN_STATIC_UPGRADE_MATRIX=true
- fi
++ # get all of our unit test filenames
++ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
- echo "***Collected DTests (j8_upgradetests_without_vnodes)***"
- set -eo pipefail && ./run_dtests.py --execute-upgrade-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw --cassandra-dir=../cassandra
- if [ -z '^upgrade_tests' ]; then
- mv /tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw /tmp/all_dtest_tests_j8_upgradetests_without_vnodes
- else
- grep -e '^upgrade_tests' /tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw > /tmp/all_dtest_tests_j8_upgradetests_without_vnodes || { echo "Filter did not match any tests! Exiting build."; exit 0; }
- fi
- set -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_upgradetests_without_vnodes > /tmp/split_dtest_tests_j8_upgradetests_without_vnodes.txt
- cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes.txt | tr '\n' ' ' > /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
- cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
- - run:
- name: Run dtests (j8_upgradetests_without_vnodes)
++ # split up the unit tests into groups based on the number of containers we have
++ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
++ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | cut -c 37-1000000 | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
++ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
++ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
++ - run:
++ name: Log Environment Information
+ command: |
- echo "cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt"
- cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
-
- source ~/env/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
- if [ -n 'RUN_STATIC_UPGRADE_MATRIX=true' ]; then
- export RUN_STATIC_UPGRADE_MATRIX=true
- fi
-
++ echo '*** id ***'
++ id
++ echo '*** cat /proc/cpuinfo ***'
++ cat /proc/cpuinfo
++ echo '*** free -m ***'
++ free -m
++ echo '*** df -m ***'
++ df -m
++ echo '*** ifconfig -a ***'
++ ifconfig -a
++ echo '*** uname -a ***'
++ uname -a
++ echo '*** mount ***'
++ mount
++ echo '*** env ***'
++ env
++ echo '*** java ***'
++ which java
+ java -version
- cd ~/cassandra-dtest
- mkdir -p /tmp/dtest
-
- echo "env: $(env)"
- echo "** done env"
- mkdir -p /tmp/results/dtests
- # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
- export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt`
- set -o pipefail && cd ~/cassandra-dtest && pytest --execute-upgrade-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_upgradetests_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
++ - run:
++ name: Run Unit Tests (testclasslist-compression)
++ command: |
++ export PATH=$JAVA_HOME/bin:$PATH
++ time mv ~/cassandra /tmp
++ cd /tmp/cassandra
++ ant testclasslist-compression -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
++ no_output_timeout: 15m
+ - store_test_results:
- path: /tmp/results
++ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
- path: /tmp/dtest
- destination: dtest_j8_upgradetests_without_vnodes
++ path: /tmp/cassandra/build/test/output
++ destination: junitxml
+ - store_artifacts:
- path: ~/cassandra-dtest/logs
- destination: dtest_j8_upgradetests_without_vnodes_logs
++ path: /tmp/cassandra/build/test/logs
++ destination: logs
+ environment:
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ workflows:
+ version: 2
+ build_and_run_tests:
+ jobs:
+ - build
+ - j8_unit_tests:
+ requires:
+ - build
+ - j8_jvm_dtests:
+ requires:
+ - build
+ - start_utests_long:
+ type: approval
+ requires:
+ - build
+ - utests_long:
+ requires:
+ - start_utests_long
+ - start_utests_compression:
+ type: approval
+ requires:
+ - build
+ - utests_compression:
+ requires:
+ - start_utests_compression
++ - start_utests_stress:
++ type: approval
++ requires:
++ - build
++ - utests_stress:
++ requires:
++ - start_utests_stress
+ - start_j8_dtests:
+ type: approval
+ requires:
+ - build
+ - j8_dtests-with-vnodes:
+ requires:
+ - start_j8_dtests
+ - j8_dtests-no-vnodes:
+ requires:
+ - start_j8_dtests
+ - start_upgrade_tests:
+ type: approval
+ requires:
+ - build
+ - j8_upgradetests-no-vnodes:
+ requires:
+ - start_upgrade_tests
+
+ # Original config.yml file:
+ # version: 2.1
+ #
+ # default_env_vars: &default_env_vars
+ # JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ # ANT_HOME: /usr/share/ant
+ # LANG: en_US.UTF-8
+ # KEEP_TEST_DIR: true
+ # DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ # PYTHONIOENCODING: utf-8
+ # PYTHONUNBUFFERED: true
+ # CASS_DRIVER_NO_EXTENSIONS: true
+ # CASS_DRIVER_NO_CYTHON: true
+ # #Skip all syncing to disk to avoid performance issues in flaky CI environments
+ # CASSANDRA_SKIP_SYNC: true
+ # DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ # DTEST_BRANCH: master
+ # CCM_MAX_HEAP_SIZE: 1024M
+ # CCM_HEAP_NEWSIZE: 256M
+ #
+ # j8_par_executor: &j8_par_executor
+ # executor:
+ # name: java8-executor
+ # #exec_resource_class: xlarge
+ # parallelism: 4
+ #
+ # j8_seq_executor: &j8_seq_executor
+ # executor:
+ # name: java8-executor
+ # #exec_resource_class: xlarge
+ # parallelism: 1 # sequential, single container tests: no parallelism benefits
+ #
+ # with_dtests_jobs: &with_dtest_jobs
+ # jobs:
+ # - build
+ # # Java 8 unit tests will be run automatically
+ # - j8_unit_tests:
+ # requires:
+ # - build
+ # - j8_jvm_dtests:
+ # requires:
+ # - build
+ # # specialized unit tests (all run on request using Java 8)
+ # - start_utests_long:
+ # type: approval
+ # requires:
+ # - build
+ # - utests_long:
+ # requires:
+ # - start_utests_long
+ # - start_utests_compression:
+ # type: approval
+ # requires:
+ # - build
+ # - utests_compression:
+ # requires:
+ # - start_utests_compression
++# - start_utests_stress:
++# type: approval
++# requires:
++# - build
++# - utests_stress:
++# requires:
++# - start_utests_stress
+ # # Java 8 dtests (on request)
+ # - start_j8_dtests:
+ # type: approval
+ # requires:
+ # - build
+ # - j8_dtests-with-vnodes:
+ # requires:
+ # - start_j8_dtests
+ # - j8_dtests-no-vnodes:
+ # requires:
+ # - start_j8_dtests
+ # # Java 8 upgrade tests
+ # - start_upgrade_tests:
+ # type: approval
+ # requires:
+ # - build
+ # - j8_upgradetests-no-vnodes:
+ # requires:
+ # - start_upgrade_tests
+ #
+ # with_dtest_jobs_only: &with_dtest_jobs_only
+ # jobs:
+ # - build
+ # - j8_dtests-with-vnodes:
+ # requires:
+ # - build
+ # - j8_dtests-no-vnodes:
+ # requires:
+ # - build
+ #
+ # workflows:
+ # version: 2
+ # build_and_run_tests: *with_dtest_jobs
+ # #build_and_run_tests: *with_dtest_jobs_only
+ #
+ # executors:
+ # java8-executor:
+ # parameters:
+ # exec_resource_class:
+ # type: string
+ # default: medium
+ # docker:
+ # - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ # resource_class: << parameters.exec_resource_class >>
+ # working_directory: ~/
+ # shell: /bin/bash -eo pipefail -l
+ # environment:
+ # <<: *default_env_vars
+ # JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ # JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ #
+ # jobs:
+ # build:
+ # executor: java8-executor
+ # parallelism: 1 # This job doesn't benefit from parallelism
+ # steps:
+ # - log_environment
+ # - clone_cassandra
+ # - build_cassandra
+ # - run_eclipse_warnings
+ # - persist_to_workspace:
+ # root: /home/cassandra
+ # paths:
+ # - cassandra
+ # - .m2
+ #
+ # j8_unit_tests:
+ # <<: *j8_par_executor
+ # steps:
+ # - attach_workspace:
+ # at: /home/cassandra
+ # - create_junit_containers
+ # - log_environment
+ # - run_parallel_junit_tests
+ #
+ # j8_jvm_dtests:
+ # <<: *j8_seq_executor
+ # steps:
+ # - attach_workspace:
+ # at: /home/cassandra
+ # - run_junit_tests:
+ # target: test-jvm-dtest-forking
+ #
+ # utests_long:
+ # <<: *j8_seq_executor
+ # steps:
+ # - attach_workspace:
+ # at: /home/cassandra
+ # - run_junit_tests:
+ # target: long-test
+ #
+ # utests_compression:
+ # <<: *j8_par_executor
+ # steps:
+ # - attach_workspace:
+ # at: /home/cassandra
+ # - create_junit_containers
+ # - log_environment
+ # - run_parallel_junit_tests:
+ # target: testclasslist-compression
+ #
++# utests_stress:
++# <<: *j8_seq_executor
++# steps:
++# - attach_workspace:
++# at: /home/cassandra
++# - run_junit_tests:
++# target: stress-test
++#
+ # j8_dtests-with-vnodes:
+ # <<: *j8_par_executor
+ # steps:
+ # - attach_workspace:
+ # at: /home/cassandra
+ # - clone_dtest
+ # - create_venv
+ # - create_dtest_containers:
+ # file_tag: j8_with_vnodes
+ # run_dtests_extra_args: '--use-vnodes --skip-resource-intensive-tests'
+ # - run_dtests:
+ # file_tag: j8_with_vnodes
+ # pytest_extra_args: '--use-vnodes --num-tokens=32 --skip-resource-intensive-tests'
+ #
+ # j8_dtests-no-vnodes:
+ # <<: *j8_par_executor
+ # steps:
+ # - attach_workspace:
+ # at: /home/cassandra
+ # - clone_dtest
+ # - create_venv
+ # - create_dtest_containers:
+ # file_tag: j8_without_vnodes
+ # run_dtests_extra_args: '--skip-resource-intensive-tests'
+ # - run_dtests:
+ # file_tag: j8_without_vnodes
+ # pytest_extra_args: '--skip-resource-intensive-tests'
+ #
+ # j8_upgradetests-no-vnodes:
+ # <<: *j8_par_executor
+ # steps:
+ # - attach_workspace:
+ # at: /home/cassandra
+ # - clone_dtest
+ # - create_venv
+ # - create_dtest_containers:
+ # file_tag: j8_upgradetests_without_vnodes
+ # run_dtests_extra_args: '--execute-upgrade-tests'
+ # extra_env_args: 'RUN_STATIC_UPGRADE_MATRIX=true'
+ # tests_filter_pattern: '^upgrade_tests'
+ # - run_dtests:
+ # file_tag: j8_upgradetests_without_vnodes
+ # extra_env_args: 'RUN_STATIC_UPGRADE_MATRIX=true'
+ # pytest_extra_args: '--execute-upgrade-tests'
+ #
+ # commands:
+ # log_environment:
+ # steps:
+ # - run:
+ # name: Log Environment Information
+ # command: |
+ # echo '*** id ***'
+ # id
+ # echo '*** cat /proc/cpuinfo ***'
+ # cat /proc/cpuinfo
+ # echo '*** free -m ***'
+ # free -m
+ # echo '*** df -m ***'
+ # df -m
+ # echo '*** ifconfig -a ***'
+ # ifconfig -a
+ # echo '*** uname -a ***'
+ # uname -a
+ # echo '*** mount ***'
+ # mount
+ # echo '*** env ***'
+ # env
+ # echo '*** java ***'
+ # which java
+ # java -version
+ #
+ # clone_cassandra:
+ # steps:
+ # - run:
+ # name: Clone Cassandra Repository (via git)
+ # command: |
+ # git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH git://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra
+ #
+ # clone_dtest:
+ # steps:
+ # - run:
+ # name: Clone Cassandra dtest Repository (via git)
+ # command: |
+ # git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ #
+ # build_cassandra:
+ # steps:
+ # - run:
+ # name: Build Cassandra
+ # command: |
+ # export PATH=$JAVA_HOME/bin:$PATH
+ # cd ~/cassandra
+ # # Loop to prevent failure due to maven-ant-tasks not downloading a jar..
+ # for x in $(seq 1 3); do
+ # ${ANT_HOME}/bin/ant clean jar
+ # RETURN=\"$?\"
+ # if [ \"${RETURN}\" -eq \"0\" ]; then
+ # break
+ # fi
+ # done
+ # # Exit, if we didn't build successfully
+ # if [ \"${RETURN}\" -ne \"0\" ]; then
+ # echo \"Build failed with exit code: ${RETURN}\"
+ # exit ${RETURN}
+ # fi
+ # no_output_timeout: 15m
+ #
+ # run_eclipse_warnings:
+ # steps:
+ # - run:
+ # name: Run eclipse-warnings
+ # command: |
+ # export PATH=$JAVA_HOME/bin:$PATH
+ # cd ~/cassandra
+ # ant eclipse-warnings
+ #
+ # create_junit_containers:
+ # steps:
+ # - run:
+ # name: Determine Unit Tests to Run
+ # command: |
+ # # reminder: this code (along with all the steps) is independently executed on every circle container
+ # # so the goal here is to get the circleci script to return the tests *this* container will run
+ # # which we do via the `circleci` cli tool.
+ #
+ # rm -fr ~/cassandra-dtest/upgrade_tests
+ # echo \"***java tests***\"
+ #
+ # # get all of our unit test filenames
+ # set -eo pipefail && circleci tests glob \"$HOME/cassandra/test/unit/**/*.java\" > /tmp/all_java_unit_tests.txt
+ #
+ # # split up the unit tests into groups based on the number of containers we have
+ # set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ # set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | cut -c 37-1000000 | grep \"Test\\.java$\" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ # echo \"** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt\"
+ # cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ #
+ # no_output_timeout: 15m
+ #
+ # run_junit_tests:
+ # parameters:
+ # target:
+ # type: string
+ # no_output_timeout:
+ # type: string
+ # default: 15m
+ # steps:
+ # - run:
+ # name: Run Unit Tests (<<parameters.target>>)
+ # # Please note that we run `clean` and therefore rebuild the project, as we can't run tests on Java 8 in case
+ # # based on Java 11 builds.
+ # command: |
+ # export PATH=$JAVA_HOME/bin:$PATH
+ # time mv ~/cassandra /tmp
+ # cd /tmp/cassandra
+ # ant clean <<parameters.target>>
+ # no_output_timeout: <<parameters.no_output_timeout>>
+ # - store_test_results:
+ # path: /tmp/cassandra/build/test/output/
+ # - store_artifacts:
+ # path: /tmp/cassandra/build/test/output
+ # destination: junitxml
+ # - store_artifacts:
+ # path: /tmp/cassandra/build/test/logs
+ # destination: logs
+ #
+ # run_parallel_junit_tests:
+ # parameters:
+ # target:
+ # type: string
+ # default: testclasslist
+ # no_output_timeout:
+ # type: string
+ # default: 15m
+ # steps:
+ # - run:
+ # name: Run Unit Tests (<<parameters.target>>)
+ # # Please note that we run `clean` and therefore rebuild the project, as we can't run tests on Java 8 in case
+ # # based on Java 11 builds.
+ # command: |
+ # export PATH=$JAVA_HOME/bin:$PATH
+ # time mv ~/cassandra /tmp
+ # cd /tmp/cassandra
+ # ant <<parameters.target>> -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
+ # no_output_timeout: <<parameters.no_output_timeout>>
+ # - store_test_results:
+ # path: /tmp/cassandra/build/test/output/
+ # - store_artifacts:
+ # path: /tmp/cassandra/build/test/output
+ # destination: junitxml
+ # - store_artifacts:
+ # path: /tmp/cassandra/build/test/logs
+ # destination: logs
+ #
+ # create_venv:
+ # steps:
+ # - run:
+ # name: Configure virtualenv and python Dependencies
+ # command: |
+ # # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ # source ~/env/bin/activate
+ # export PATH=$JAVA_HOME/bin:$PATH
+ # pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
+ # pip3 freeze
+ #
+ # create_dtest_containers:
+ # parameters:
+ # file_tag:
+ # type: string
+ # run_dtests_extra_args:
+ # type: string
+ # default: ''
+ # extra_env_args:
+ # type: string
+ # default: ''
+ # tests_filter_pattern:
+ # type: string
+ # default: ''
+ # steps:
+ # - run:
+ # name: Determine Tests to Run (<<parameters.file_tag>>)
+ # no_output_timeout: 5m
+ # command: |
+ # # reminder: this code (along with all the steps) is independently executed on every circle container
+ # # so the goal here is to get the circleci script to return the tests *this* container will run
+ # # which we do via the `circleci` cli tool.
+ #
+ # cd cassandra-dtest
+ # source ~/env/bin/activate
+ # export PATH=$JAVA_HOME/bin:$PATH
+ #
+ # if [ -n '<<parameters.extra_env_args>>' ]; then
+ # export <<parameters.extra_env_args>>
+ # fi
+ #
+ # echo \"***Collected DTests (<<parameters.file_tag>>)***\"
+ # set -eo pipefail && ./run_dtests.py <<parameters.run_dtests_extra_args>> --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_<<parameters.file_tag>>_raw --cassandra-dir=../cassandra
+ # if [ -z '<<parameters.tests_filter_pattern>>' ]; then
+ # mv /tmp/all_dtest_tests_<<parameters.file_tag>>_raw /tmp/all_dtest_tests_<<parameters.file_tag>>
+ # else
+ # grep -e '<<parameters.tests_filter_pattern>>' /tmp/all_dtest_tests_<<parameters.file_tag>>_raw > /tmp/all_dtest_tests_<<parameters.file_tag>> || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }
+ # fi
+ # set -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_<<parameters.file_tag>> > /tmp/split_dtest_tests_<<parameters.file_tag>>.txt
+ # cat /tmp/split_dtest_tests_<<parameters.file_tag>>.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_<<parameters.file_tag>>_final.txt
+ # cat /tmp/split_dtest_tests_<<parameters.file_tag>>_final.txt
+ #
+ # run_dtests:
+ # parameters:
+ # file_tag:
+ # type: string
+ # pytest_extra_args:
+ # type: string
+ # default: ''
+ # extra_env_args:
+ # type: string
+ # default: ''
+ # steps:
+ # - run:
+ # name: Run dtests (<<parameters.file_tag>>)
+ # no_output_timeout: 15m
+ # command: |
+ # echo \"cat /tmp/split_dtest_tests_<<parameters.file_tag>>_final.txt\"
+ # cat /tmp/split_dtest_tests_<<parameters.file_tag>>_final.txt
+ #
+ # source ~/env/bin/activate
+ # export PATH=$JAVA_HOME/bin:$PATH
+ # if [ -n '<<parameters.extra_env_args>>' ]; then
+ # export <<parameters.extra_env_args>>
+ # fi
+ #
+ # java -version
+ # cd ~/cassandra-dtest
+ # mkdir -p /tmp/dtest
+ #
+ # echo \"env: $(env)\"
+ # echo \"** done env\"
+ # mkdir -p /tmp/results/dtests
+ # # we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ # export SPLIT_TESTS=`cat /tmp/split_dtest_tests_<<parameters.file_tag>>_final.txt`
+ # set -o pipefail && cd ~/cassandra-dtest && pytest <<parameters.pytest_extra_args>> --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_<<parameters.file_tag>>.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ # - store_test_results:
+ # path: /tmp/results
+ # - store_artifacts:
+ # path: /tmp/dtest
+ # destination: dtest_<<parameters.file_tag>>
+ # - store_artifacts:
+ # path: ~/cassandra-dtest/logs
+ # destination: dtest_<<parameters.file_tag>>_logs
diff --cc CHANGES.txt
index 2e1234b,9945c76..716a6dd
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,11 -1,10 +1,12 @@@
-3.0.19
+3.11.5
+ * Add flag to disable SASI indexes, and warnings on creation (CASSANDRA-14866)
+Merged from 3.0:
* Improve `nodetool status -r` speed (CASSANDRA-14847)
* Improve merkle tree size and time on heap (CASSANDRA-14096)
- * Add missing commands to nodetool-completion (CASSANDRA-14916)
+ * Add missing commands to nodetool_completion (CASSANDRA-14916)
* Anti-compaction temporarily corrupts sstable state for readers (CASSANDRA-15004)
- Merged from 2.2:
+Merged from 2.2:
+ * Refactor Circle CI configuration (CASSANDRA-14806)
* Fixing invalid CQL in security documentation (CASSANDRA-15020)
* Multi-version in-JVM dtests (CASSANDRA-14937)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org