You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by ma...@apache.org on 2020/01/16 10:14:01 UTC
[cassandra] branch cassandra-3.0 updated: Run in-jvm upgrade dtests
in circle ci
This is an automated email from the ASF dual-hosted git repository.
marcuse pushed a commit to branch cassandra-3.0
in repository https://gitbox.apache.org/repos/asf/cassandra.git
The following commit(s) were added to refs/heads/cassandra-3.0 by this push:
new 5f57212 Run in-jvm upgrade dtests in circle ci
5f57212 is described below
commit 5f5721226788a9db73ad37a48bd02a66c84b7e5c
Author: Marcus Eriksson <ma...@apache.org>
AuthorDate: Wed Jan 15 10:40:03 2020 +0100
Run in-jvm upgrade dtests in circle ci
Patch by marcuse; reviewed by Alex Petrov for CASSANDRA-15506
---
.circleci/config-2_1.yml | 74 ++
.circleci/config.yml | 1004 ++++++-------------
.circleci/config.yml.HIGHRES | 1006 +++++++-------------
.circleci/config.yml.LOWRES | 1004 ++++++-------------
CHANGES.txt | 1 +
build.xml | 35 +
.../cassandra/distributed/test/TestLocator.java | 8 +-
7 files changed, 1069 insertions(+), 2063 deletions(-)
diff --git a/.circleci/config-2_1.yml b/.circleci/config-2_1.yml
index 859100d..a4410fe 100644
--- a/.circleci/config-2_1.yml
+++ b/.circleci/config-2_1.yml
@@ -54,6 +54,16 @@ with_dtests_jobs: &with_dtest_jobs
- utests_compression:
requires:
- start_utests_compression
+ - start_jvm_upgrade_dtest:
+ type: approval
+ requires:
+ - build
+ - dtest_jars_build:
+ requires:
+ - start_jvm_upgrade_dtest
+ - j8_jvm_upgrade_dtests:
+ requires:
+ - dtest_jars_build
# Java 8 dtests (on request)
- start_j8_dtests:
type: approval
@@ -120,6 +130,18 @@ jobs:
- cassandra
- .m2
+ dtest_jars_build:
+ executor: java8-executor
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - build_cassandra_dtest_jars
+ - persist_to_workspace:
+ root: /home/cassandra
+ paths:
+ - dtest_jars
+
j8_unit_tests:
<<: *j8_par_executor
steps:
@@ -137,6 +159,14 @@ jobs:
- run_junit_tests:
target: test-jvm-dtest-forking
+ j8_jvm_upgrade_dtests:
+ <<: *j8_seq_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run_junit_tests:
+ target: test-jvm-upgrade-dtest-forking
+
utests_long:
<<: *j8_seq_executor
steps:
@@ -262,6 +292,47 @@ commands:
fi
no_output_timeout: 15m
+ build_cassandra_dtest_jars:
+ steps:
+ - run:
+ name: Build Cassandra DTest jars
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ cd ~/cassandra
+ git remote add apache git://github.com/apache/cassandra.git
+ for branch in cassandra-2.2 cassandra-3.0 cassandra-3.11 trunk; do
+ # check out the correct cassandra version:
+ git remote set-branches --add apache '$branch'
+ git fetch --depth 1 apache $branch
+ git checkout $branch
+ # Loop to prevent failure due to maven-ant-tasks not downloading a jar..
+ for x in $(seq 1 3); do
+ ${ANT_HOME}/bin/ant clean jar dtest-jar
+ RETURN="$?"
+ if [ "${RETURN}" -eq "0" ]; then
+ break
+ fi
+ done
+ # Exit, if we didn't build successfully
+ if [ "${RETURN}" -ne "0" ]; then
+ echo "Build failed with exit code: ${RETURN}"
+ exit ${RETURN}
+ fi
+ done
+ # and build the dtest-jar for the branch under test
+ git checkout origin/$CIRCLE_BRANCH
+ for x in $(seq 1 3); do
+ ${ANT_HOME}/bin/ant clean jar dtest-jar
+ RETURN="$?"
+ if [ "${RETURN}" -eq "0" ]; then
+ break
+ fi
+ done
+ mkdir ~/dtest_jars
+ cp build/dtest*.jar ~/dtest_jars
+ ls -l ~/dtest_jars
+ no_output_timeout: 15m
+
run_eclipse_warnings:
steps:
- run:
@@ -310,6 +381,9 @@ commands:
export PATH=$JAVA_HOME/bin:$PATH
time mv ~/cassandra /tmp
cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
ant clean <<parameters.target>>
no_output_timeout: <<parameters.no_output_timeout>>
- store_test_results:
diff --git a/.circleci/config.yml b/.circleci/config.yml
index 0d56e23..d80b78d 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -1,5 +1,51 @@
version: 2
jobs:
+ j8_jvm_upgrade_dtests:
+ docker:
+ - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Run Unit Tests (test-jvm-upgrade-dtest-forking)
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ ant clean test-jvm-upgrade-dtest-forking
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
build:
docker:
- image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
@@ -81,6 +127,163 @@ jobs:
- CCM_HEAP_NEWSIZE: 256M
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_dtests-no-vnodes:
+ docker:
+ - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-inten [...]
+ - run:
+ name: Run dtests (j8_without_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the [...]
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j8_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j8_without_vnodes_logs
+ environment:
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_upgradetests-no-vnodes:
+ docker:
+ - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_upgradetests_without_vnodes)
+ no_output_timeout: 5m
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ cd cassandra-dtest
+ source ~/env/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+
+ if [ -n 'RUN_STATIC_UPGRADE_MATRIX=true' ]; then
+ export RUN_STATIC_UPGRADE_MATRIX=true
+ fi
+
+ echo "***Collected DTests (j8_upgradetests_without_vnodes)***"
+ set -eo pipefail && ./run_dtests.py --execute-upgrade-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw --cassandra-dir=../cassandra
+ if [ -z '^upgrade_tests' ]; then
+ mv /tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw /tmp/all_dtest_tests_j8_upgradetests_without_vnodes
+ else
+ grep -e '^upgrade_tests' /tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw > /tmp/all_dtest_tests_j8_upgradetests_without_vnodes || { echo "Filter did not match any tests! Exiting build."; exit 0; }
+ fi
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_upgradetests_without_vnodes > /tmp/split_dtest_tests_j8_upgradetests_without_vnodes.txt
+ cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes.txt | tr '\n' ' ' > /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
+ cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
+ - run:
+ name: Run dtests (j8_upgradetests_without_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
+
+ source ~/env/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'RUN_STATIC_UPGRADE_MATRIX=true' ]; then
+ export RUN_STATIC_UPGRADE_MATRIX=true
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --execute-upgrade-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_upgradetests_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j8_upgradetests_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j8_upgradetests_without_vnodes_logs
+ environment:
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_unit_tests:
docker:
- image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
@@ -165,6 +368,64 @@ jobs:
- CCM_HEAP_NEWSIZE: 256M
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_dtests-with-vnodes:
+ docker:
+ - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-reso [...]
+ - run:
+ name: Run dtests (j8_with_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit [...]
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j8_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j8_with_vnodes_logs
+ environment:
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_jvm_dtests:
docker:
- image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
@@ -181,6 +442,9 @@ jobs:
export PATH=$JAVA_HOME/bin:$PATH
time mv ~/cassandra /tmp
cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
ant clean test-jvm-dtest-forking
no_output_timeout: 15m
- store_test_results:
@@ -224,6 +488,9 @@ jobs:
export PATH=$JAVA_HOME/bin:$PATH
time mv ~/cassandra /tmp
cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
ant clean long-test
no_output_timeout: 15m
- store_test_results:
@@ -335,255 +602,58 @@ jobs:
- CCM_HEAP_NEWSIZE: 256M
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_dtests-with-vnodes:
- docker:
- - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
- resource_class: medium
- working_directory: ~/
- shell: /bin/bash -eo pipefail -l
- parallelism: 4
- steps:
- - attach_workspace:
- at: /home/cassandra
- - run:
- name: Clone Cassandra dtest Repository (via git)
- command: |
- git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- - run:
- name: Configure virtualenv and python Dependencies
- command: |
- # note, this should be super quick as all dependencies should be pre-installed in the docker image
- # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
- # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
- # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
- source ~/env/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
- pip3 freeze
- - run:
- name: Determine Tests to Run (j8_with_vnodes)
- no_output_timeout: 5m
- command: "# reminder: this code (along with all the steps) is independently\
- \ executed on every circle container\n# so the goal here is to get the circleci\
- \ script to return the tests *this* container will run\n# which we do via\
- \ the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\n\
- export PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\n\
- echo \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py\
- \ --use-vnodes --skip-resource-intensive-tests --dtest-print-tests-only\
- \ --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\n\
- if [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\n\
- else\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes\
- \ || { echo \"Filter did not match any tests! Exiting build.\"; exit 0;\
- \ }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname\
- \ /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\n\
- cat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\
- cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n"
- - run:
- name: Run dtests (j8_with_vnodes)
- no_output_timeout: 15m
- command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat\
- \ /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env/bin/activate\n\
- export PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\n\
- java -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env:\
- \ $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need\
- \ the \"set -o pipefail\" here so that the exit code that circleci will\
- \ actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat\
- \ /tmp/split_dtest_tests_j8_with_vnodes_final.txt`\nset -o pipefail && cd\
- \ ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests\
- \ --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml\
- \ -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS\
- \ 2>&1 | tee /tmp/dtest/stdout.txt\n"
- - store_test_results:
- path: /tmp/results
- - store_artifacts:
- path: /tmp/dtest
- destination: dtest_j8_with_vnodes
- - store_artifacts:
- path: ~/cassandra-dtest/logs
- destination: dtest_j8_with_vnodes_logs
- environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - ANT_HOME: /usr/share/ant
- - LANG: en_US.UTF-8
- - KEEP_TEST_DIR: true
- - DEFAULT_DIR: /home/cassandra/cassandra-dtest
- - PYTHONIOENCODING: utf-8
- - PYTHONUNBUFFERED: true
- - CASS_DRIVER_NO_EXTENSIONS: true
- - CASS_DRIVER_NO_CYTHON: true
- - CASSANDRA_SKIP_SYNC: true
- - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: master
- - CCM_MAX_HEAP_SIZE: 1024M
- - CCM_HEAP_NEWSIZE: 256M
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_dtests-no-vnodes:
+ dtest_jars_build:
docker:
- image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
- parallelism: 4
- steps:
- - attach_workspace:
- at: /home/cassandra
- - run:
- name: Clone Cassandra dtest Repository (via git)
- command: |
- git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- - run:
- name: Configure virtualenv and python Dependencies
- command: |
- # note, this should be super quick as all dependencies should be pre-installed in the docker image
- # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
- # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
- # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
- source ~/env/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
- pip3 freeze
- - run:
- name: Determine Tests to Run (j8_without_vnodes)
- no_output_timeout: 5m
- command: "# reminder: this code (along with all the steps) is independently\
- \ executed on every circle container\n# so the goal here is to get the circleci\
- \ script to return the tests *this* container will run\n# which we do via\
- \ the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\n\
- export PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\n\
- echo \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail &&\
- \ ./run_dtests.py --skip-resource-intensive-tests --dtest-print-tests-only\
- \ --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw\
- \ --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw\
- \ /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw\
- \ > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match\
- \ any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci\
- \ tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes\
- \ > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt\
- \ | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\
- cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n"
- - run:
- name: Run dtests (j8_without_vnodes)
- no_output_timeout: 15m
- command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\n\
- cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env/bin/activate\n\
- export PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\n\
- java -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env:\
- \ $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need\
- \ the \"set -o pipefail\" here so that the exit code that circleci will\
- \ actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat\
- \ /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nset -o pipefail &&\
- \ cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"\
- INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml\
- \ -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS\
- \ 2>&1 | tee /tmp/dtest/stdout.txt\n"
- - store_test_results:
- path: /tmp/results
- - store_artifacts:
- path: /tmp/dtest
- destination: dtest_j8_without_vnodes
- - store_artifacts:
- path: ~/cassandra-dtest/logs
- destination: dtest_j8_without_vnodes_logs
- environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - ANT_HOME: /usr/share/ant
- - LANG: en_US.UTF-8
- - KEEP_TEST_DIR: true
- - DEFAULT_DIR: /home/cassandra/cassandra-dtest
- - PYTHONIOENCODING: utf-8
- - PYTHONUNBUFFERED: true
- - CASS_DRIVER_NO_EXTENSIONS: true
- - CASS_DRIVER_NO_CYTHON: true
- - CASSANDRA_SKIP_SYNC: true
- - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: master
- - CCM_MAX_HEAP_SIZE: 1024M
- - CCM_HEAP_NEWSIZE: 256M
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_upgradetests-no-vnodes:
- docker:
- - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
- resource_class: medium
- working_directory: ~/
- shell: /bin/bash -eo pipefail -l
- parallelism: 4
+ parallelism: 1
steps:
- attach_workspace:
at: /home/cassandra
- run:
- name: Clone Cassandra dtest Repository (via git)
+ name: Build Cassandra DTest jars
command: |
- git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- - run:
- name: Configure virtualenv and python Dependencies
- command: |
- # note, this should be super quick as all dependencies should be pre-installed in the docker image
- # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
- # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
- # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
- source ~/env/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
- pip3 freeze
- - run:
- name: Determine Tests to Run (j8_upgradetests_without_vnodes)
- no_output_timeout: 5m
- command: |
- # reminder: this code (along with all the steps) is independently executed on every circle container
- # so the goal here is to get the circleci script to return the tests *this* container will run
- # which we do via the `circleci` cli tool.
-
- cd cassandra-dtest
- source ~/env/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
-
- if [ -n 'RUN_STATIC_UPGRADE_MATRIX=true' ]; then
- export RUN_STATIC_UPGRADE_MATRIX=true
- fi
-
- echo "***Collected DTests (j8_upgradetests_without_vnodes)***"
- set -eo pipefail && ./run_dtests.py --execute-upgrade-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw --cassandra-dir=../cassandra
- if [ -z '^upgrade_tests' ]; then
- mv /tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw /tmp/all_dtest_tests_j8_upgradetests_without_vnodes
- else
- grep -e '^upgrade_tests' /tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw > /tmp/all_dtest_tests_j8_upgradetests_without_vnodes || { echo "Filter did not match any tests! Exiting build."; exit 0; }
- fi
- set -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_upgradetests_without_vnodes > /tmp/split_dtest_tests_j8_upgradetests_without_vnodes.txt
- cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes.txt | tr '\n' ' ' > /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
- cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
- - run:
- name: Run dtests (j8_upgradetests_without_vnodes)
+ cd ~/cassandra
+ git remote add apache git://github.com/apache/cassandra.git
+ for branch in cassandra-2.2 cassandra-3.0 cassandra-3.11 trunk; do
+ # check out the correct cassandra version:
+ git remote set-branches --add apache '$branch'
+ git fetch --depth 1 apache $branch
+ git checkout $branch
+ # Loop to prevent failure due to maven-ant-tasks not downloading a jar..
+ for x in $(seq 1 3); do
+ ${ANT_HOME}/bin/ant clean jar dtest-jar
+ RETURN="$?"
+ if [ "${RETURN}" -eq "0" ]; then
+ break
+ fi
+ done
+ # Exit, if we didn't build successfully
+ if [ "${RETURN}" -ne "0" ]; then
+ echo "Build failed with exit code: ${RETURN}"
+ exit ${RETURN}
+ fi
+ done
+ # and build the dtest-jar for the branch under test
+ git checkout origin/$CIRCLE_BRANCH
+ for x in $(seq 1 3); do
+ ${ANT_HOME}/bin/ant clean jar dtest-jar
+ RETURN="$?"
+ if [ "${RETURN}" -eq "0" ]; then
+ break
+ fi
+ done
+ mkdir ~/dtest_jars
+ cp build/dtest*.jar ~/dtest_jars
+ ls -l ~/dtest_jars
no_output_timeout: 15m
- command: |
- echo "cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt"
- cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
-
- source ~/env/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
- if [ -n 'RUN_STATIC_UPGRADE_MATRIX=true' ]; then
- export RUN_STATIC_UPGRADE_MATRIX=true
- fi
-
- java -version
- cd ~/cassandra-dtest
- mkdir -p /tmp/dtest
-
- echo "env: $(env)"
- echo "** done env"
- mkdir -p /tmp/results/dtests
- # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
- export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt`
- set -o pipefail && cd ~/cassandra-dtest && pytest --execute-upgrade-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_upgradetests_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
- - store_test_results:
- path: /tmp/results
- - store_artifacts:
- path: /tmp/dtest
- destination: dtest_j8_upgradetests_without_vnodes
- - store_artifacts:
- path: ~/cassandra-dtest/logs
- destination: dtest_j8_upgradetests_without_vnodes_logs
+ - persist_to_workspace:
+ root: /home/cassandra
+ paths:
+ - dtest_jars
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
@@ -606,6 +676,16 @@ workflows:
build_and_run_tests:
jobs:
- build
+ - start_jvm_upgrade_dtest:
+ type: approval
+ requires:
+ - build
+ - dtest_jars_build:
+ requires:
+ - start_jvm_upgrade_dtest
+ - j8_jvm_upgrade_dtests:
+ requires:
+ - dtest_jars_build
- j8_unit_tests:
requires:
- build
@@ -643,453 +723,3 @@ workflows:
- j8_upgradetests-no-vnodes:
requires:
- start_upgrade_tests
-
-# Original config.yml file:
-# version: 2.1
-#
-# default_env_vars: &default_env_vars
-# JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
-# ANT_HOME: /usr/share/ant
-# LANG: en_US.UTF-8
-# KEEP_TEST_DIR: true
-# DEFAULT_DIR: /home/cassandra/cassandra-dtest
-# PYTHONIOENCODING: utf-8
-# PYTHONUNBUFFERED: true
-# CASS_DRIVER_NO_EXTENSIONS: true
-# CASS_DRIVER_NO_CYTHON: true
-# #Skip all syncing to disk to avoid performance issues in flaky CI environments
-# CASSANDRA_SKIP_SYNC: true
-# DTEST_REPO: git://github.com/apache/cassandra-dtest.git
-# DTEST_BRANCH: master
-# CCM_MAX_HEAP_SIZE: 1024M
-# CCM_HEAP_NEWSIZE: 256M
-#
-# j8_par_executor: &j8_par_executor
-# executor:
-# name: java8-executor
-# #exec_resource_class: xlarge
-# parallelism: 4
-#
-# j8_seq_executor: &j8_seq_executor
-# executor:
-# name: java8-executor
-# #exec_resource_class: xlarge
-# parallelism: 1 # sequential, single container tests: no parallelism benefits
-#
-# with_dtests_jobs: &with_dtest_jobs
-# jobs:
-# - build
-# # Java 8 unit tests will be run automatically
-# - j8_unit_tests:
-# requires:
-# - build
-# - j8_jvm_dtests:
-# requires:
-# - build
-# # specialized unit tests (all run on request using Java 8)
-# - start_utests_long:
-# type: approval
-# requires:
-# - build
-# - utests_long:
-# requires:
-# - start_utests_long
-# - start_utests_compression:
-# type: approval
-# requires:
-# - build
-# - utests_compression:
-# requires:
-# - start_utests_compression
-# # Java 8 dtests (on request)
-# - start_j8_dtests:
-# type: approval
-# requires:
-# - build
-# - j8_dtests-with-vnodes:
-# requires:
-# - start_j8_dtests
-# - j8_dtests-no-vnodes:
-# requires:
-# - start_j8_dtests
-# # Java 8 upgrade tests
-# - start_upgrade_tests:
-# type: approval
-# requires:
-# - build
-# - j8_upgradetests-no-vnodes:
-# requires:
-# - start_upgrade_tests
-#
-# with_dtest_jobs_only: &with_dtest_jobs_only
-# jobs:
-# - build
-# - j8_dtests-with-vnodes:
-# requires:
-# - build
-# - j8_dtests-no-vnodes:
-# requires:
-# - build
-#
-# workflows:
-# version: 2
-# build_and_run_tests: *with_dtest_jobs
-# #build_and_run_tests: *with_dtest_jobs_only
-#
-# executors:
-# java8-executor:
-# parameters:
-# exec_resource_class:
-# type: string
-# default: medium
-# docker:
-# - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
-# resource_class: << parameters.exec_resource_class >>
-# working_directory: ~/
-# shell: /bin/bash -eo pipefail -l
-# environment:
-# <<: *default_env_vars
-# JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
-# JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
-#
-# jobs:
-# build:
-# executor: java8-executor
-# parallelism: 1 # This job doesn't benefit from parallelism
-# steps:
-# - log_environment
-# - clone_cassandra
-# - build_cassandra
-# - run_eclipse_warnings
-# - persist_to_workspace:
-# root: /home/cassandra
-# paths:
-# - cassandra
-# - .m2
-#
-# j8_unit_tests:
-# <<: *j8_par_executor
-# steps:
-# - attach_workspace:
-# at: /home/cassandra
-# - create_junit_containers
-# - log_environment
-# - run_parallel_junit_tests
-#
-# j8_jvm_dtests:
-# <<: *j8_seq_executor
-# steps:
-# - attach_workspace:
-# at: /home/cassandra
-# - run_junit_tests:
-# target: test-jvm-dtest-forking
-#
-# utests_long:
-# <<: *j8_seq_executor
-# steps:
-# - attach_workspace:
-# at: /home/cassandra
-# - run_junit_tests:
-# target: long-test
-#
-# utests_compression:
-# <<: *j8_par_executor
-# steps:
-# - attach_workspace:
-# at: /home/cassandra
-# - create_junit_containers
-# - log_environment
-# - run_parallel_junit_tests:
-# target: testclasslist-compression
-#
-# j8_dtests-with-vnodes:
-# <<: *j8_par_executor
-# steps:
-# - attach_workspace:
-# at: /home/cassandra
-# - clone_dtest
-# - create_venv
-# - create_dtest_containers:
-# file_tag: j8_with_vnodes
-# run_dtests_extra_args: '--use-vnodes --skip-resource-intensive-tests'
-# - run_dtests:
-# file_tag: j8_with_vnodes
-# pytest_extra_args: '--use-vnodes --num-tokens=32 --skip-resource-intensive-tests'
-#
-# j8_dtests-no-vnodes:
-# <<: *j8_par_executor
-# steps:
-# - attach_workspace:
-# at: /home/cassandra
-# - clone_dtest
-# - create_venv
-# - create_dtest_containers:
-# file_tag: j8_without_vnodes
-# run_dtests_extra_args: '--skip-resource-intensive-tests'
-# - run_dtests:
-# file_tag: j8_without_vnodes
-# pytest_extra_args: '--skip-resource-intensive-tests'
-#
-# j8_upgradetests-no-vnodes:
-# <<: *j8_par_executor
-# steps:
-# - attach_workspace:
-# at: /home/cassandra
-# - clone_dtest
-# - create_venv
-# - create_dtest_containers:
-# file_tag: j8_upgradetests_without_vnodes
-# run_dtests_extra_args: '--execute-upgrade-tests'
-# extra_env_args: 'RUN_STATIC_UPGRADE_MATRIX=true'
-# tests_filter_pattern: '^upgrade_tests'
-# - run_dtests:
-# file_tag: j8_upgradetests_without_vnodes
-# extra_env_args: 'RUN_STATIC_UPGRADE_MATRIX=true'
-# pytest_extra_args: '--execute-upgrade-tests'
-#
-# commands:
-# log_environment:
-# steps:
-# - run:
-# name: Log Environment Information
-# command: |
-# echo '*** id ***'
-# id
-# echo '*** cat /proc/cpuinfo ***'
-# cat /proc/cpuinfo
-# echo '*** free -m ***'
-# free -m
-# echo '*** df -m ***'
-# df -m
-# echo '*** ifconfig -a ***'
-# ifconfig -a
-# echo '*** uname -a ***'
-# uname -a
-# echo '*** mount ***'
-# mount
-# echo '*** env ***'
-# env
-# echo '*** java ***'
-# which java
-# java -version
-#
-# clone_cassandra:
-# steps:
-# - run:
-# name: Clone Cassandra Repository (via git)
-# command: |
-# git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH git://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra
-#
-# clone_dtest:
-# steps:
-# - run:
-# name: Clone Cassandra dtest Repository (via git)
-# command: |
-# git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
-#
-# build_cassandra:
-# steps:
-# - run:
-# name: Build Cassandra
-# command: |
-# export PATH=$JAVA_HOME/bin:$PATH
-# cd ~/cassandra
-# # Loop to prevent failure due to maven-ant-tasks not downloading a jar..
-# for x in $(seq 1 3); do
-# ${ANT_HOME}/bin/ant clean jar
-# RETURN=\"$?\"
-# if [ \"${RETURN}\" -eq \"0\" ]; then
-# break
-# fi
-# done
-# # Exit, if we didn't build successfully
-# if [ \"${RETURN}\" -ne \"0\" ]; then
-# echo \"Build failed with exit code: ${RETURN}\"
-# exit ${RETURN}
-# fi
-# no_output_timeout: 15m
-#
-# run_eclipse_warnings:
-# steps:
-# - run:
-# name: Run eclipse-warnings
-# command: |
-# export PATH=$JAVA_HOME/bin:$PATH
-# cd ~/cassandra
-# ant eclipse-warnings
-#
-# create_junit_containers:
-# steps:
-# - run:
-# name: Determine Unit Tests to Run
-# command: |
-# # reminder: this code (along with all the steps) is independently executed on every circle container
-# # so the goal here is to get the circleci script to return the tests *this* container will run
-# # which we do via the `circleci` cli tool.
-#
-# rm -fr ~/cassandra-dtest/upgrade_tests
-# echo \"***java tests***\"
-#
-# # get all of our unit test filenames
-# set -eo pipefail && circleci tests glob \"$HOME/cassandra/test/unit/**/*.java\" > /tmp/all_java_unit_tests.txt
-#
-# # split up the unit tests into groups based on the number of containers we have
-# set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
-# set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | cut -c 37-1000000 | grep \"Test\\.java$\" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
-# echo \"** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt\"
-# cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
-#
-# no_output_timeout: 15m
-#
-# run_junit_tests:
-# parameters:
-# target:
-# type: string
-# no_output_timeout:
-# type: string
-# default: 15m
-# steps:
-# - run:
-# name: Run Unit Tests (<<parameters.target>>)
-# # Please note that we run `clean` and therefore rebuild the project, as we can't run tests on Java 8 in case
-# # based on Java 11 builds.
-# command: |
-# export PATH=$JAVA_HOME/bin:$PATH
-# time mv ~/cassandra /tmp
-# cd /tmp/cassandra
-# ant clean <<parameters.target>>
-# no_output_timeout: <<parameters.no_output_timeout>>
-# - store_test_results:
-# path: /tmp/cassandra/build/test/output/
-# - store_artifacts:
-# path: /tmp/cassandra/build/test/output
-# destination: junitxml
-# - store_artifacts:
-# path: /tmp/cassandra/build/test/logs
-# destination: logs
-#
-# run_parallel_junit_tests:
-# parameters:
-# target:
-# type: string
-# default: testclasslist
-# no_output_timeout:
-# type: string
-# default: 15m
-# steps:
-# - run:
-# name: Run Unit Tests (<<parameters.target>>)
-# # Please note that we run `clean` and therefore rebuild the project, as we can't run tests on Java 8 in case
-# # based on Java 11 builds.
-# command: |
-# export PATH=$JAVA_HOME/bin:$PATH
-# time mv ~/cassandra /tmp
-# cd /tmp/cassandra
-# ant <<parameters.target>> -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
-# no_output_timeout: <<parameters.no_output_timeout>>
-# - store_test_results:
-# path: /tmp/cassandra/build/test/output/
-# - store_artifacts:
-# path: /tmp/cassandra/build/test/output
-# destination: junitxml
-# - store_artifacts:
-# path: /tmp/cassandra/build/test/logs
-# destination: logs
-#
-# create_venv:
-# steps:
-# - run:
-# name: Configure virtualenv and python Dependencies
-# command: |
-# # note, this should be super quick as all dependencies should be pre-installed in the docker image
-# # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
-# # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
-# # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
-# source ~/env/bin/activate
-# export PATH=$JAVA_HOME/bin:$PATH
-# pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
-# pip3 freeze
-#
-# create_dtest_containers:
-# parameters:
-# file_tag:
-# type: string
-# run_dtests_extra_args:
-# type: string
-# default: ''
-# extra_env_args:
-# type: string
-# default: ''
-# tests_filter_pattern:
-# type: string
-# default: ''
-# steps:
-# - run:
-# name: Determine Tests to Run (<<parameters.file_tag>>)
-# no_output_timeout: 5m
-# command: |
-# # reminder: this code (along with all the steps) is independently executed on every circle container
-# # so the goal here is to get the circleci script to return the tests *this* container will run
-# # which we do via the `circleci` cli tool.
-#
-# cd cassandra-dtest
-# source ~/env/bin/activate
-# export PATH=$JAVA_HOME/bin:$PATH
-#
-# if [ -n '<<parameters.extra_env_args>>' ]; then
-# export <<parameters.extra_env_args>>
-# fi
-#
-# echo \"***Collected DTests (<<parameters.file_tag>>)***\"
-# set -eo pipefail && ./run_dtests.py <<parameters.run_dtests_extra_args>> --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_<<parameters.file_tag>>_raw --cassandra-dir=../cassandra
-# if [ -z '<<parameters.tests_filter_pattern>>' ]; then
-# mv /tmp/all_dtest_tests_<<parameters.file_tag>>_raw /tmp/all_dtest_tests_<<parameters.file_tag>>
-# else
-# grep -e '<<parameters.tests_filter_pattern>>' /tmp/all_dtest_tests_<<parameters.file_tag>>_raw > /tmp/all_dtest_tests_<<parameters.file_tag>> || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }
-# fi
-# set -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_<<parameters.file_tag>> > /tmp/split_dtest_tests_<<parameters.file_tag>>.txt
-# cat /tmp/split_dtest_tests_<<parameters.file_tag>>.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_<<parameters.file_tag>>_final.txt
-# cat /tmp/split_dtest_tests_<<parameters.file_tag>>_final.txt
-#
-# run_dtests:
-# parameters:
-# file_tag:
-# type: string
-# pytest_extra_args:
-# type: string
-# default: ''
-# extra_env_args:
-# type: string
-# default: ''
-# steps:
-# - run:
-# name: Run dtests (<<parameters.file_tag>>)
-# no_output_timeout: 15m
-# command: |
-# echo \"cat /tmp/split_dtest_tests_<<parameters.file_tag>>_final.txt\"
-# cat /tmp/split_dtest_tests_<<parameters.file_tag>>_final.txt
-#
-# source ~/env/bin/activate
-# export PATH=$JAVA_HOME/bin:$PATH
-# if [ -n '<<parameters.extra_env_args>>' ]; then
-# export <<parameters.extra_env_args>>
-# fi
-#
-# java -version
-# cd ~/cassandra-dtest
-# mkdir -p /tmp/dtest
-#
-# echo \"env: $(env)\"
-# echo \"** done env\"
-# mkdir -p /tmp/results/dtests
-# # we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
-# export SPLIT_TESTS=`cat /tmp/split_dtest_tests_<<parameters.file_tag>>_final.txt`
-# set -o pipefail && cd ~/cassandra-dtest && pytest <<parameters.pytest_extra_args>> --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_<<parameters.file_tag>>.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
-# - store_test_results:
-# path: /tmp/results
-# - store_artifacts:
-# path: /tmp/dtest
-# destination: dtest_<<parameters.file_tag>>
-# - store_artifacts:
-# path: ~/cassandra-dtest/logs
-# destination: dtest_<<parameters.file_tag>>_logs
\ No newline at end of file
diff --git a/.circleci/config.yml.HIGHRES b/.circleci/config.yml.HIGHRES
index ab38b45..94ca5b2 100644
--- a/.circleci/config.yml.HIGHRES
+++ b/.circleci/config.yml.HIGHRES
@@ -1,5 +1,51 @@
version: 2
jobs:
+ j8_jvm_upgrade_dtests:
+ docker:
+ - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Run Unit Tests (test-jvm-upgrade-dtest-forking)
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ ant clean test-jvm-upgrade-dtest-forking
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 2048M
+ - CCM_HEAP_NEWSIZE: 512M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
build:
docker:
- image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
@@ -81,6 +127,163 @@ jobs:
- CCM_HEAP_NEWSIZE: 512M
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_dtests-no-vnodes:
+ docker:
+ - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-inten [...]
+ - run:
+ name: Run dtests (j8_without_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the [...]
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j8_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j8_without_vnodes_logs
+ environment:
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 2048M
+ - CCM_HEAP_NEWSIZE: 512M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_upgradetests-no-vnodes:
+ docker:
+ - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_upgradetests_without_vnodes)
+ no_output_timeout: 5m
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ cd cassandra-dtest
+ source ~/env/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+
+ if [ -n 'RUN_STATIC_UPGRADE_MATRIX=true' ]; then
+ export RUN_STATIC_UPGRADE_MATRIX=true
+ fi
+
+ echo "***Collected DTests (j8_upgradetests_without_vnodes)***"
+ set -eo pipefail && ./run_dtests.py --execute-upgrade-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw --cassandra-dir=../cassandra
+ if [ -z '^upgrade_tests' ]; then
+ mv /tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw /tmp/all_dtest_tests_j8_upgradetests_without_vnodes
+ else
+ grep -e '^upgrade_tests' /tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw > /tmp/all_dtest_tests_j8_upgradetests_without_vnodes || { echo "Filter did not match any tests! Exiting build."; exit 0; }
+ fi
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_upgradetests_without_vnodes > /tmp/split_dtest_tests_j8_upgradetests_without_vnodes.txt
+ cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes.txt | tr '\n' ' ' > /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
+ cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
+ - run:
+ name: Run dtests (j8_upgradetests_without_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
+
+ source ~/env/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'RUN_STATIC_UPGRADE_MATRIX=true' ]; then
+ export RUN_STATIC_UPGRADE_MATRIX=true
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --execute-upgrade-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_upgradetests_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j8_upgradetests_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j8_upgradetests_without_vnodes_logs
+ environment:
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 2048M
+ - CCM_HEAP_NEWSIZE: 512M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_unit_tests:
docker:
- image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
@@ -165,6 +368,64 @@ jobs:
- CCM_HEAP_NEWSIZE: 512M
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_dtests-with-vnodes:
+ docker:
+ - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-reso [...]
+ - run:
+ name: Run dtests (j8_with_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit [...]
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j8_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j8_with_vnodes_logs
+ environment:
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 2048M
+ - CCM_HEAP_NEWSIZE: 512M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_jvm_dtests:
docker:
- image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
@@ -181,6 +442,9 @@ jobs:
export PATH=$JAVA_HOME/bin:$PATH
time mv ~/cassandra /tmp
cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
ant clean test-jvm-dtest-forking
no_output_timeout: 15m
- store_test_results:
@@ -224,6 +488,9 @@ jobs:
export PATH=$JAVA_HOME/bin:$PATH
time mv ~/cassandra /tmp
cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
ant clean long-test
no_output_timeout: 15m
- store_test_results:
@@ -335,255 +602,58 @@ jobs:
- CCM_HEAP_NEWSIZE: 512M
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_dtests-with-vnodes:
- docker:
- - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
- resource_class: xlarge
- working_directory: ~/
- shell: /bin/bash -eo pipefail -l
- parallelism: 100
- steps:
- - attach_workspace:
- at: /home/cassandra
- - run:
- name: Clone Cassandra dtest Repository (via git)
- command: |
- git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- - run:
- name: Configure virtualenv and python Dependencies
- command: |
- # note, this should be super quick as all dependencies should be pre-installed in the docker image
- # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
- # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
- # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
- source ~/env/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
- pip3 freeze
- - run:
- name: Determine Tests to Run (j8_with_vnodes)
- no_output_timeout: 5m
- command: "# reminder: this code (along with all the steps) is independently\
- \ executed on every circle container\n# so the goal here is to get the circleci\
- \ script to return the tests *this* container will run\n# which we do via\
- \ the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\n\
- export PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\n\
- echo \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py\
- \ --use-vnodes --skip-resource-intensive-tests --dtest-print-tests-only\
- \ --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\n\
- if [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\n\
- else\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes\
- \ || { echo \"Filter did not match any tests! Exiting build.\"; exit 0;\
- \ }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname\
- \ /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\n\
- cat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\
- cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n"
- - run:
- name: Run dtests (j8_with_vnodes)
- no_output_timeout: 15m
- command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat\
- \ /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env/bin/activate\n\
- export PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\n\
- java -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env:\
- \ $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need\
- \ the \"set -o pipefail\" here so that the exit code that circleci will\
- \ actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat\
- \ /tmp/split_dtest_tests_j8_with_vnodes_final.txt`\nset -o pipefail && cd\
- \ ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests\
- \ --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml\
- \ -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS\
- \ 2>&1 | tee /tmp/dtest/stdout.txt\n"
- - store_test_results:
- path: /tmp/results
- - store_artifacts:
- path: /tmp/dtest
- destination: dtest_j8_with_vnodes
- - store_artifacts:
- path: ~/cassandra-dtest/logs
- destination: dtest_j8_with_vnodes_logs
- environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - ANT_HOME: /usr/share/ant
- - LANG: en_US.UTF-8
- - KEEP_TEST_DIR: true
- - DEFAULT_DIR: /home/cassandra/cassandra-dtest
- - PYTHONIOENCODING: utf-8
- - PYTHONUNBUFFERED: true
- - CASS_DRIVER_NO_EXTENSIONS: true
- - CASS_DRIVER_NO_CYTHON: true
- - CASSANDRA_SKIP_SYNC: true
- - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: master
- - CCM_MAX_HEAP_SIZE: 2048M
- - CCM_HEAP_NEWSIZE: 512M
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_dtests-no-vnodes:
- docker:
- - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
- resource_class: xlarge
- working_directory: ~/
- shell: /bin/bash -eo pipefail -l
- parallelism: 100
- steps:
- - attach_workspace:
- at: /home/cassandra
- - run:
- name: Clone Cassandra dtest Repository (via git)
- command: |
- git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- - run:
- name: Configure virtualenv and python Dependencies
- command: |
- # note, this should be super quick as all dependencies should be pre-installed in the docker image
- # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
- # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
- # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
- source ~/env/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
- pip3 freeze
- - run:
- name: Determine Tests to Run (j8_without_vnodes)
- no_output_timeout: 5m
- command: "# reminder: this code (along with all the steps) is independently\
- \ executed on every circle container\n# so the goal here is to get the circleci\
- \ script to return the tests *this* container will run\n# which we do via\
- \ the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\n\
- export PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\n\
- echo \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail &&\
- \ ./run_dtests.py --skip-resource-intensive-tests --dtest-print-tests-only\
- \ --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw\
- \ --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw\
- \ /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw\
- \ > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match\
- \ any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci\
- \ tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes\
- \ > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt\
- \ | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\
- cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n"
- - run:
- name: Run dtests (j8_without_vnodes)
- no_output_timeout: 15m
- command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\n\
- cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env/bin/activate\n\
- export PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\n\
- java -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env:\
- \ $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need\
- \ the \"set -o pipefail\" here so that the exit code that circleci will\
- \ actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat\
- \ /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nset -o pipefail &&\
- \ cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"\
- INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml\
- \ -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS\
- \ 2>&1 | tee /tmp/dtest/stdout.txt\n"
- - store_test_results:
- path: /tmp/results
- - store_artifacts:
- path: /tmp/dtest
- destination: dtest_j8_without_vnodes
- - store_artifacts:
- path: ~/cassandra-dtest/logs
- destination: dtest_j8_without_vnodes_logs
- environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - ANT_HOME: /usr/share/ant
- - LANG: en_US.UTF-8
- - KEEP_TEST_DIR: true
- - DEFAULT_DIR: /home/cassandra/cassandra-dtest
- - PYTHONIOENCODING: utf-8
- - PYTHONUNBUFFERED: true
- - CASS_DRIVER_NO_EXTENSIONS: true
- - CASS_DRIVER_NO_CYTHON: true
- - CASSANDRA_SKIP_SYNC: true
- - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: master
- - CCM_MAX_HEAP_SIZE: 2048M
- - CCM_HEAP_NEWSIZE: 512M
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_upgradetests-no-vnodes:
+ dtest_jars_build:
docker:
- image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
- resource_class: xlarge
+ resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
- parallelism: 100
+ parallelism: 1
steps:
- attach_workspace:
at: /home/cassandra
- run:
- name: Clone Cassandra dtest Repository (via git)
- command: |
- git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- - run:
- name: Configure virtualenv and python Dependencies
- command: |
- # note, this should be super quick as all dependencies should be pre-installed in the docker image
- # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
- # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
- # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
- source ~/env/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
- pip3 freeze
- - run:
- name: Determine Tests to Run (j8_upgradetests_without_vnodes)
- no_output_timeout: 5m
+ name: Build Cassandra DTest jars
command: |
- # reminder: this code (along with all the steps) is independently executed on every circle container
- # so the goal here is to get the circleci script to return the tests *this* container will run
- # which we do via the `circleci` cli tool.
-
- cd cassandra-dtest
- source ~/env/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
-
- if [ -n 'RUN_STATIC_UPGRADE_MATRIX=true' ]; then
- export RUN_STATIC_UPGRADE_MATRIX=true
- fi
-
- echo "***Collected DTests (j8_upgradetests_without_vnodes)***"
- set -eo pipefail && ./run_dtests.py --execute-upgrade-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw --cassandra-dir=../cassandra
- if [ -z '^upgrade_tests' ]; then
- mv /tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw /tmp/all_dtest_tests_j8_upgradetests_without_vnodes
- else
- grep -e '^upgrade_tests' /tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw > /tmp/all_dtest_tests_j8_upgradetests_without_vnodes || { echo "Filter did not match any tests! Exiting build."; exit 0; }
- fi
- set -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_upgradetests_without_vnodes > /tmp/split_dtest_tests_j8_upgradetests_without_vnodes.txt
- cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes.txt | tr '\n' ' ' > /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
- cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
- - run:
- name: Run dtests (j8_upgradetests_without_vnodes)
+ cd ~/cassandra
+ git remote add apache git://github.com/apache/cassandra.git
+ for branch in cassandra-2.2 cassandra-3.0 cassandra-3.11 trunk; do
+ # check out the correct cassandra version:
+ git remote set-branches --add apache '$branch'
+ git fetch --depth 1 apache $branch
+ git checkout $branch
+ # Loop to prevent failure due to maven-ant-tasks not downloading a jar..
+ for x in $(seq 1 3); do
+ ${ANT_HOME}/bin/ant clean jar dtest-jar
+ RETURN="$?"
+ if [ "${RETURN}" -eq "0" ]; then
+ break
+ fi
+ done
+ # Exit, if we didn't build successfully
+ if [ "${RETURN}" -ne "0" ]; then
+ echo "Build failed with exit code: ${RETURN}"
+ exit ${RETURN}
+ fi
+ done
+ # and build the dtest-jar for the branch under test
+ git checkout origin/$CIRCLE_BRANCH
+ for x in $(seq 1 3); do
+ ${ANT_HOME}/bin/ant clean jar dtest-jar
+ RETURN="$?"
+ if [ "${RETURN}" -eq "0" ]; then
+ break
+ fi
+ done
+ mkdir ~/dtest_jars
+ cp build/dtest*.jar ~/dtest_jars
+ ls -l ~/dtest_jars
no_output_timeout: 15m
- command: |
- echo "cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt"
- cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
-
- source ~/env/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
- if [ -n 'RUN_STATIC_UPGRADE_MATRIX=true' ]; then
- export RUN_STATIC_UPGRADE_MATRIX=true
- fi
-
- java -version
- cd ~/cassandra-dtest
- mkdir -p /tmp/dtest
-
- echo "env: $(env)"
- echo "** done env"
- mkdir -p /tmp/results/dtests
- # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
- export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt`
- set -o pipefail && cd ~/cassandra-dtest && pytest --execute-upgrade-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_upgradetests_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
- - store_test_results:
- path: /tmp/results
- - store_artifacts:
- path: /tmp/dtest
- destination: dtest_j8_upgradetests_without_vnodes
- - store_artifacts:
- path: ~/cassandra-dtest/logs
- destination: dtest_j8_upgradetests_without_vnodes_logs
+ - persist_to_workspace:
+ root: /home/cassandra
+ paths:
+ - dtest_jars
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
@@ -606,6 +676,16 @@ workflows:
build_and_run_tests:
jobs:
- build
+ - start_jvm_upgrade_dtest:
+ type: approval
+ requires:
+ - build
+ - dtest_jars_build:
+ requires:
+ - start_jvm_upgrade_dtest
+ - j8_jvm_upgrade_dtests:
+ requires:
+ - dtest_jars_build
- j8_unit_tests:
requires:
- build
@@ -643,453 +723,3 @@ workflows:
- j8_upgradetests-no-vnodes:
requires:
- start_upgrade_tests
-
-# Original config.yml file:
-# version: 2.1
-#
-# default_env_vars: &default_env_vars
-# JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
-# ANT_HOME: /usr/share/ant
-# LANG: en_US.UTF-8
-# KEEP_TEST_DIR: true
-# DEFAULT_DIR: /home/cassandra/cassandra-dtest
-# PYTHONIOENCODING: utf-8
-# PYTHONUNBUFFERED: true
-# CASS_DRIVER_NO_EXTENSIONS: true
-# CASS_DRIVER_NO_CYTHON: true
-# #Skip all syncing to disk to avoid performance issues in flaky CI environments
-# CASSANDRA_SKIP_SYNC: true
-# DTEST_REPO: git://github.com/apache/cassandra-dtest.git
-# DTEST_BRANCH: master
-# CCM_MAX_HEAP_SIZE: 2048M
-# CCM_HEAP_NEWSIZE: 512M
-#
-# j8_par_executor: &j8_par_executor
-# executor:
-# name: java8-executor
-# exec_resource_class: xlarge
-# parallelism: 100
-#
-# j8_seq_executor: &j8_seq_executor
-# executor:
-# name: java8-executor
-# exec_resource_class: xlarge
-# parallelism: 1 # sequential, single container tests: no parallelism benefits
-#
-# with_dtests_jobs: &with_dtest_jobs
-# jobs:
-# - build
-# # Java 8 unit tests will be run automatically
-# - j8_unit_tests:
-# requires:
-# - build
-# - j8_jvm_dtests:
-# requires:
-# - build
-# # specialized unit tests (all run on request using Java 8)
-# - start_utests_long:
-# type: approval
-# requires:
-# - build
-# - utests_long:
-# requires:
-# - start_utests_long
-# - start_utests_compression:
-# type: approval
-# requires:
-# - build
-# - utests_compression:
-# requires:
-# - start_utests_compression
-# # Java 8 dtests (on request)
-# - start_j8_dtests:
-# type: approval
-# requires:
-# - build
-# - j8_dtests-with-vnodes:
-# requires:
-# - start_j8_dtests
-# - j8_dtests-no-vnodes:
-# requires:
-# - start_j8_dtests
-# # Java 8 upgrade tests
-# - start_upgrade_tests:
-# type: approval
-# requires:
-# - build
-# - j8_upgradetests-no-vnodes:
-# requires:
-# - start_upgrade_tests
-#
-# with_dtest_jobs_only: &with_dtest_jobs_only
-# jobs:
-# - build
-# - j8_dtests-with-vnodes:
-# requires:
-# - build
-# - j8_dtests-no-vnodes:
-# requires:
-# - build
-#
-# workflows:
-# version: 2
-# build_and_run_tests: *with_dtest_jobs
-# #build_and_run_tests: *with_dtest_jobs_only
-#
-# executors:
-# java8-executor:
-# parameters:
-# exec_resource_class:
-# type: string
-# default: medium
-# docker:
-# - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
-# resource_class: << parameters.exec_resource_class >>
-# working_directory: ~/
-# shell: /bin/bash -eo pipefail -l
-# environment:
-# <<: *default_env_vars
-# JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
-# JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
-#
-# jobs:
-# build:
-# executor: java8-executor
-# parallelism: 1 # This job doesn't benefit from parallelism
-# steps:
-# - log_environment
-# - clone_cassandra
-# - build_cassandra
-# - run_eclipse_warnings
-# - persist_to_workspace:
-# root: /home/cassandra
-# paths:
-# - cassandra
-# - .m2
-#
-# j8_unit_tests:
-# <<: *j8_par_executor
-# steps:
-# - attach_workspace:
-# at: /home/cassandra
-# - create_junit_containers
-# - log_environment
-# - run_parallel_junit_tests
-#
-# j8_jvm_dtests:
-# <<: *j8_seq_executor
-# steps:
-# - attach_workspace:
-# at: /home/cassandra
-# - run_junit_tests:
-# target: test-jvm-dtest-forking
-#
-# utests_long:
-# <<: *j8_seq_executor
-# steps:
-# - attach_workspace:
-# at: /home/cassandra
-# - run_junit_tests:
-# target: long-test
-#
-# utests_compression:
-# <<: *j8_par_executor
-# steps:
-# - attach_workspace:
-# at: /home/cassandra
-# - create_junit_containers
-# - log_environment
-# - run_parallel_junit_tests:
-# target: testclasslist-compression
-#
-# j8_dtests-with-vnodes:
-# <<: *j8_par_executor
-# steps:
-# - attach_workspace:
-# at: /home/cassandra
-# - clone_dtest
-# - create_venv
-# - create_dtest_containers:
-# file_tag: j8_with_vnodes
-# run_dtests_extra_args: '--use-vnodes --skip-resource-intensive-tests'
-# - run_dtests:
-# file_tag: j8_with_vnodes
-# pytest_extra_args: '--use-vnodes --num-tokens=32 --skip-resource-intensive-tests'
-#
-# j8_dtests-no-vnodes:
-# <<: *j8_par_executor
-# steps:
-# - attach_workspace:
-# at: /home/cassandra
-# - clone_dtest
-# - create_venv
-# - create_dtest_containers:
-# file_tag: j8_without_vnodes
-# run_dtests_extra_args: '--skip-resource-intensive-tests'
-# - run_dtests:
-# file_tag: j8_without_vnodes
-# pytest_extra_args: '--skip-resource-intensive-tests'
-#
-# j8_upgradetests-no-vnodes:
-# <<: *j8_par_executor
-# steps:
-# - attach_workspace:
-# at: /home/cassandra
-# - clone_dtest
-# - create_venv
-# - create_dtest_containers:
-# file_tag: j8_upgradetests_without_vnodes
-# run_dtests_extra_args: '--execute-upgrade-tests'
-# extra_env_args: 'RUN_STATIC_UPGRADE_MATRIX=true'
-# tests_filter_pattern: '^upgrade_tests'
-# - run_dtests:
-# file_tag: j8_upgradetests_without_vnodes
-# extra_env_args: 'RUN_STATIC_UPGRADE_MATRIX=true'
-# pytest_extra_args: '--execute-upgrade-tests'
-#
-# commands:
-# log_environment:
-# steps:
-# - run:
-# name: Log Environment Information
-# command: |
-# echo '*** id ***'
-# id
-# echo '*** cat /proc/cpuinfo ***'
-# cat /proc/cpuinfo
-# echo '*** free -m ***'
-# free -m
-# echo '*** df -m ***'
-# df -m
-# echo '*** ifconfig -a ***'
-# ifconfig -a
-# echo '*** uname -a ***'
-# uname -a
-# echo '*** mount ***'
-# mount
-# echo '*** env ***'
-# env
-# echo '*** java ***'
-# which java
-# java -version
-#
-# clone_cassandra:
-# steps:
-# - run:
-# name: Clone Cassandra Repository (via git)
-# command: |
-# git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH git://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra
-#
-# clone_dtest:
-# steps:
-# - run:
-# name: Clone Cassandra dtest Repository (via git)
-# command: |
-# git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
-#
-# build_cassandra:
-# steps:
-# - run:
-# name: Build Cassandra
-# command: |
-# export PATH=$JAVA_HOME/bin:$PATH
-# cd ~/cassandra
-# # Loop to prevent failure due to maven-ant-tasks not downloading a jar..
-# for x in $(seq 1 3); do
-# ${ANT_HOME}/bin/ant clean jar
-# RETURN=\"$?\"
-# if [ \"${RETURN}\" -eq \"0\" ]; then
-# break
-# fi
-# done
-# # Exit, if we didn't build successfully
-# if [ \"${RETURN}\" -ne \"0\" ]; then
-# echo \"Build failed with exit code: ${RETURN}\"
-# exit ${RETURN}
-# fi
-# no_output_timeout: 15m
-#
-# run_eclipse_warnings:
-# steps:
-# - run:
-# name: Run eclipse-warnings
-# command: |
-# export PATH=$JAVA_HOME/bin:$PATH
-# cd ~/cassandra
-# ant eclipse-warnings
-#
-# create_junit_containers:
-# steps:
-# - run:
-# name: Determine Unit Tests to Run
-# command: |
-# # reminder: this code (along with all the steps) is independently executed on every circle container
-# # so the goal here is to get the circleci script to return the tests *this* container will run
-# # which we do via the `circleci` cli tool.
-#
-# rm -fr ~/cassandra-dtest/upgrade_tests
-# echo \"***java tests***\"
-#
-# # get all of our unit test filenames
-# set -eo pipefail && circleci tests glob \"$HOME/cassandra/test/unit/**/*.java\" > /tmp/all_java_unit_tests.txt
-#
-# # split up the unit tests into groups based on the number of containers we have
-# set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
-# set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | cut -c 37-1000000 | grep \"Test\\.java$\" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
-# echo \"** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt\"
-# cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
-#
-# no_output_timeout: 15m
-#
-# run_junit_tests:
-# parameters:
-# target:
-# type: string
-# no_output_timeout:
-# type: string
-# default: 15m
-# steps:
-# - run:
-# name: Run Unit Tests (<<parameters.target>>)
-# # Please note that we run `clean` and therefore rebuild the project, as we can't run tests on Java 8 in case
-# # based on Java 11 builds.
-# command: |
-# export PATH=$JAVA_HOME/bin:$PATH
-# time mv ~/cassandra /tmp
-# cd /tmp/cassandra
-# ant clean <<parameters.target>>
-# no_output_timeout: <<parameters.no_output_timeout>>
-# - store_test_results:
-# path: /tmp/cassandra/build/test/output/
-# - store_artifacts:
-# path: /tmp/cassandra/build/test/output
-# destination: junitxml
-# - store_artifacts:
-# path: /tmp/cassandra/build/test/logs
-# destination: logs
-#
-# run_parallel_junit_tests:
-# parameters:
-# target:
-# type: string
-# default: testclasslist
-# no_output_timeout:
-# type: string
-# default: 15m
-# steps:
-# - run:
-# name: Run Unit Tests (<<parameters.target>>)
-# # Please note that we run `clean` and therefore rebuild the project, as we can't run tests on Java 8 in case
-# # based on Java 11 builds.
-# command: |
-# export PATH=$JAVA_HOME/bin:$PATH
-# time mv ~/cassandra /tmp
-# cd /tmp/cassandra
-# ant <<parameters.target>> -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
-# no_output_timeout: <<parameters.no_output_timeout>>
-# - store_test_results:
-# path: /tmp/cassandra/build/test/output/
-# - store_artifacts:
-# path: /tmp/cassandra/build/test/output
-# destination: junitxml
-# - store_artifacts:
-# path: /tmp/cassandra/build/test/logs
-# destination: logs
-#
-# create_venv:
-# steps:
-# - run:
-# name: Configure virtualenv and python Dependencies
-# command: |
-# # note, this should be super quick as all dependencies should be pre-installed in the docker image
-# # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
-# # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
-# # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
-# source ~/env/bin/activate
-# export PATH=$JAVA_HOME/bin:$PATH
-# pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
-# pip3 freeze
-#
-# create_dtest_containers:
-# parameters:
-# file_tag:
-# type: string
-# run_dtests_extra_args:
-# type: string
-# default: ''
-# extra_env_args:
-# type: string
-# default: ''
-# tests_filter_pattern:
-# type: string
-# default: ''
-# steps:
-# - run:
-# name: Determine Tests to Run (<<parameters.file_tag>>)
-# no_output_timeout: 5m
-# command: |
-# # reminder: this code (along with all the steps) is independently executed on every circle container
-# # so the goal here is to get the circleci script to return the tests *this* container will run
-# # which we do via the `circleci` cli tool.
-#
-# cd cassandra-dtest
-# source ~/env/bin/activate
-# export PATH=$JAVA_HOME/bin:$PATH
-#
-# if [ -n '<<parameters.extra_env_args>>' ]; then
-# export <<parameters.extra_env_args>>
-# fi
-#
-# echo \"***Collected DTests (<<parameters.file_tag>>)***\"
-# set -eo pipefail && ./run_dtests.py <<parameters.run_dtests_extra_args>> --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_<<parameters.file_tag>>_raw --cassandra-dir=../cassandra
-# if [ -z '<<parameters.tests_filter_pattern>>' ]; then
-# mv /tmp/all_dtest_tests_<<parameters.file_tag>>_raw /tmp/all_dtest_tests_<<parameters.file_tag>>
-# else
-# grep -e '<<parameters.tests_filter_pattern>>' /tmp/all_dtest_tests_<<parameters.file_tag>>_raw > /tmp/all_dtest_tests_<<parameters.file_tag>> || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }
-# fi
-# set -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_<<parameters.file_tag>> > /tmp/split_dtest_tests_<<parameters.file_tag>>.txt
-# cat /tmp/split_dtest_tests_<<parameters.file_tag>>.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_<<parameters.file_tag>>_final.txt
-# cat /tmp/split_dtest_tests_<<parameters.file_tag>>_final.txt
-#
-# run_dtests:
-# parameters:
-# file_tag:
-# type: string
-# pytest_extra_args:
-# type: string
-# default: ''
-# extra_env_args:
-# type: string
-# default: ''
-# steps:
-# - run:
-# name: Run dtests (<<parameters.file_tag>>)
-# no_output_timeout: 15m
-# command: |
-# echo \"cat /tmp/split_dtest_tests_<<parameters.file_tag>>_final.txt\"
-# cat /tmp/split_dtest_tests_<<parameters.file_tag>>_final.txt
-#
-# source ~/env/bin/activate
-# export PATH=$JAVA_HOME/bin:$PATH
-# if [ -n '<<parameters.extra_env_args>>' ]; then
-# export <<parameters.extra_env_args>>
-# fi
-#
-# java -version
-# cd ~/cassandra-dtest
-# mkdir -p /tmp/dtest
-#
-# echo \"env: $(env)\"
-# echo \"** done env\"
-# mkdir -p /tmp/results/dtests
-# # we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
-# export SPLIT_TESTS=`cat /tmp/split_dtest_tests_<<parameters.file_tag>>_final.txt`
-# set -o pipefail && cd ~/cassandra-dtest && pytest <<parameters.pytest_extra_args>> --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_<<parameters.file_tag>>.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
-# - store_test_results:
-# path: /tmp/results
-# - store_artifacts:
-# path: /tmp/dtest
-# destination: dtest_<<parameters.file_tag>>
-# - store_artifacts:
-# path: ~/cassandra-dtest/logs
-# destination: dtest_<<parameters.file_tag>>_logs
\ No newline at end of file
diff --git a/.circleci/config.yml.LOWRES b/.circleci/config.yml.LOWRES
index 0d56e23..d80b78d 100644
--- a/.circleci/config.yml.LOWRES
+++ b/.circleci/config.yml.LOWRES
@@ -1,5 +1,51 @@
version: 2
jobs:
+ j8_jvm_upgrade_dtests:
+ docker:
+ - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Run Unit Tests (test-jvm-upgrade-dtest-forking)
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ ant clean test-jvm-upgrade-dtest-forking
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
build:
docker:
- image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
@@ -81,6 +127,163 @@ jobs:
- CCM_HEAP_NEWSIZE: 256M
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_dtests-no-vnodes:
+ docker:
+ - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-inten [...]
+ - run:
+ name: Run dtests (j8_without_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the [...]
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j8_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j8_without_vnodes_logs
+ environment:
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_upgradetests-no-vnodes:
+ docker:
+ - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_upgradetests_without_vnodes)
+ no_output_timeout: 5m
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ cd cassandra-dtest
+ source ~/env/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+
+ if [ -n 'RUN_STATIC_UPGRADE_MATRIX=true' ]; then
+ export RUN_STATIC_UPGRADE_MATRIX=true
+ fi
+
+ echo "***Collected DTests (j8_upgradetests_without_vnodes)***"
+ set -eo pipefail && ./run_dtests.py --execute-upgrade-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw --cassandra-dir=../cassandra
+ if [ -z '^upgrade_tests' ]; then
+ mv /tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw /tmp/all_dtest_tests_j8_upgradetests_without_vnodes
+ else
+ grep -e '^upgrade_tests' /tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw > /tmp/all_dtest_tests_j8_upgradetests_without_vnodes || { echo "Filter did not match any tests! Exiting build."; exit 0; }
+ fi
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_upgradetests_without_vnodes > /tmp/split_dtest_tests_j8_upgradetests_without_vnodes.txt
+ cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes.txt | tr '\n' ' ' > /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
+ cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
+ - run:
+ name: Run dtests (j8_upgradetests_without_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
+
+ source ~/env/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'RUN_STATIC_UPGRADE_MATRIX=true' ]; then
+ export RUN_STATIC_UPGRADE_MATRIX=true
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --execute-upgrade-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_upgradetests_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j8_upgradetests_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j8_upgradetests_without_vnodes_logs
+ environment:
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_unit_tests:
docker:
- image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
@@ -165,6 +368,64 @@ jobs:
- CCM_HEAP_NEWSIZE: 256M
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_dtests-with-vnodes:
+ docker:
+ - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-reso [...]
+ - run:
+ name: Run dtests (j8_with_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit [...]
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j8_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j8_with_vnodes_logs
+ environment:
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_jvm_dtests:
docker:
- image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
@@ -181,6 +442,9 @@ jobs:
export PATH=$JAVA_HOME/bin:$PATH
time mv ~/cassandra /tmp
cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
ant clean test-jvm-dtest-forking
no_output_timeout: 15m
- store_test_results:
@@ -224,6 +488,9 @@ jobs:
export PATH=$JAVA_HOME/bin:$PATH
time mv ~/cassandra /tmp
cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
ant clean long-test
no_output_timeout: 15m
- store_test_results:
@@ -335,255 +602,58 @@ jobs:
- CCM_HEAP_NEWSIZE: 256M
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_dtests-with-vnodes:
- docker:
- - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
- resource_class: medium
- working_directory: ~/
- shell: /bin/bash -eo pipefail -l
- parallelism: 4
- steps:
- - attach_workspace:
- at: /home/cassandra
- - run:
- name: Clone Cassandra dtest Repository (via git)
- command: |
- git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- - run:
- name: Configure virtualenv and python Dependencies
- command: |
- # note, this should be super quick as all dependencies should be pre-installed in the docker image
- # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
- # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
- # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
- source ~/env/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
- pip3 freeze
- - run:
- name: Determine Tests to Run (j8_with_vnodes)
- no_output_timeout: 5m
- command: "# reminder: this code (along with all the steps) is independently\
- \ executed on every circle container\n# so the goal here is to get the circleci\
- \ script to return the tests *this* container will run\n# which we do via\
- \ the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\n\
- export PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\n\
- echo \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py\
- \ --use-vnodes --skip-resource-intensive-tests --dtest-print-tests-only\
- \ --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\n\
- if [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\n\
- else\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes\
- \ || { echo \"Filter did not match any tests! Exiting build.\"; exit 0;\
- \ }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname\
- \ /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\n\
- cat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\
- cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n"
- - run:
- name: Run dtests (j8_with_vnodes)
- no_output_timeout: 15m
- command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat\
- \ /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env/bin/activate\n\
- export PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\n\
- java -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env:\
- \ $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need\
- \ the \"set -o pipefail\" here so that the exit code that circleci will\
- \ actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat\
- \ /tmp/split_dtest_tests_j8_with_vnodes_final.txt`\nset -o pipefail && cd\
- \ ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests\
- \ --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml\
- \ -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS\
- \ 2>&1 | tee /tmp/dtest/stdout.txt\n"
- - store_test_results:
- path: /tmp/results
- - store_artifacts:
- path: /tmp/dtest
- destination: dtest_j8_with_vnodes
- - store_artifacts:
- path: ~/cassandra-dtest/logs
- destination: dtest_j8_with_vnodes_logs
- environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - ANT_HOME: /usr/share/ant
- - LANG: en_US.UTF-8
- - KEEP_TEST_DIR: true
- - DEFAULT_DIR: /home/cassandra/cassandra-dtest
- - PYTHONIOENCODING: utf-8
- - PYTHONUNBUFFERED: true
- - CASS_DRIVER_NO_EXTENSIONS: true
- - CASS_DRIVER_NO_CYTHON: true
- - CASSANDRA_SKIP_SYNC: true
- - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: master
- - CCM_MAX_HEAP_SIZE: 1024M
- - CCM_HEAP_NEWSIZE: 256M
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_dtests-no-vnodes:
+ dtest_jars_build:
docker:
- image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
- parallelism: 4
- steps:
- - attach_workspace:
- at: /home/cassandra
- - run:
- name: Clone Cassandra dtest Repository (via git)
- command: |
- git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- - run:
- name: Configure virtualenv and python Dependencies
- command: |
- # note, this should be super quick as all dependencies should be pre-installed in the docker image
- # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
- # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
- # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
- source ~/env/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
- pip3 freeze
- - run:
- name: Determine Tests to Run (j8_without_vnodes)
- no_output_timeout: 5m
- command: "# reminder: this code (along with all the steps) is independently\
- \ executed on every circle container\n# so the goal here is to get the circleci\
- \ script to return the tests *this* container will run\n# which we do via\
- \ the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\n\
- export PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\n\
- echo \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail &&\
- \ ./run_dtests.py --skip-resource-intensive-tests --dtest-print-tests-only\
- \ --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw\
- \ --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw\
- \ /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw\
- \ > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match\
- \ any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci\
- \ tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes\
- \ > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt\
- \ | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\
- cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n"
- - run:
- name: Run dtests (j8_without_vnodes)
- no_output_timeout: 15m
- command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\n\
- cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env/bin/activate\n\
- export PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\n\
- java -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env:\
- \ $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need\
- \ the \"set -o pipefail\" here so that the exit code that circleci will\
- \ actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat\
- \ /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nset -o pipefail &&\
- \ cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"\
- INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml\
- \ -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS\
- \ 2>&1 | tee /tmp/dtest/stdout.txt\n"
- - store_test_results:
- path: /tmp/results
- - store_artifacts:
- path: /tmp/dtest
- destination: dtest_j8_without_vnodes
- - store_artifacts:
- path: ~/cassandra-dtest/logs
- destination: dtest_j8_without_vnodes_logs
- environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - ANT_HOME: /usr/share/ant
- - LANG: en_US.UTF-8
- - KEEP_TEST_DIR: true
- - DEFAULT_DIR: /home/cassandra/cassandra-dtest
- - PYTHONIOENCODING: utf-8
- - PYTHONUNBUFFERED: true
- - CASS_DRIVER_NO_EXTENSIONS: true
- - CASS_DRIVER_NO_CYTHON: true
- - CASSANDRA_SKIP_SYNC: true
- - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: master
- - CCM_MAX_HEAP_SIZE: 1024M
- - CCM_HEAP_NEWSIZE: 256M
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_upgradetests-no-vnodes:
- docker:
- - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
- resource_class: medium
- working_directory: ~/
- shell: /bin/bash -eo pipefail -l
- parallelism: 4
+ parallelism: 1
steps:
- attach_workspace:
at: /home/cassandra
- run:
- name: Clone Cassandra dtest Repository (via git)
+ name: Build Cassandra DTest jars
command: |
- git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- - run:
- name: Configure virtualenv and python Dependencies
- command: |
- # note, this should be super quick as all dependencies should be pre-installed in the docker image
- # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
- # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
- # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
- source ~/env/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
- pip3 freeze
- - run:
- name: Determine Tests to Run (j8_upgradetests_without_vnodes)
- no_output_timeout: 5m
- command: |
- # reminder: this code (along with all the steps) is independently executed on every circle container
- # so the goal here is to get the circleci script to return the tests *this* container will run
- # which we do via the `circleci` cli tool.
-
- cd cassandra-dtest
- source ~/env/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
-
- if [ -n 'RUN_STATIC_UPGRADE_MATRIX=true' ]; then
- export RUN_STATIC_UPGRADE_MATRIX=true
- fi
-
- echo "***Collected DTests (j8_upgradetests_without_vnodes)***"
- set -eo pipefail && ./run_dtests.py --execute-upgrade-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw --cassandra-dir=../cassandra
- if [ -z '^upgrade_tests' ]; then
- mv /tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw /tmp/all_dtest_tests_j8_upgradetests_without_vnodes
- else
- grep -e '^upgrade_tests' /tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw > /tmp/all_dtest_tests_j8_upgradetests_without_vnodes || { echo "Filter did not match any tests! Exiting build."; exit 0; }
- fi
- set -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_upgradetests_without_vnodes > /tmp/split_dtest_tests_j8_upgradetests_without_vnodes.txt
- cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes.txt | tr '\n' ' ' > /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
- cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
- - run:
- name: Run dtests (j8_upgradetests_without_vnodes)
+ cd ~/cassandra
+ git remote add apache git://github.com/apache/cassandra.git
+ for branch in cassandra-2.2 cassandra-3.0 cassandra-3.11 trunk; do
+ # check out the correct cassandra version:
+ git remote set-branches --add apache '$branch'
+ git fetch --depth 1 apache $branch
+ git checkout $branch
+ # Loop to prevent failure due to maven-ant-tasks not downloading a jar..
+ for x in $(seq 1 3); do
+ ${ANT_HOME}/bin/ant clean jar dtest-jar
+ RETURN="$?"
+ if [ "${RETURN}" -eq "0" ]; then
+ break
+ fi
+ done
+ # Exit, if we didn't build successfully
+ if [ "${RETURN}" -ne "0" ]; then
+ echo "Build failed with exit code: ${RETURN}"
+ exit ${RETURN}
+ fi
+ done
+ # and build the dtest-jar for the branch under test
+ git checkout origin/$CIRCLE_BRANCH
+ for x in $(seq 1 3); do
+ ${ANT_HOME}/bin/ant clean jar dtest-jar
+ RETURN="$?"
+ if [ "${RETURN}" -eq "0" ]; then
+ break
+ fi
+ done
+ mkdir ~/dtest_jars
+ cp build/dtest*.jar ~/dtest_jars
+ ls -l ~/dtest_jars
no_output_timeout: 15m
- command: |
- echo "cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt"
- cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
-
- source ~/env/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
- if [ -n 'RUN_STATIC_UPGRADE_MATRIX=true' ]; then
- export RUN_STATIC_UPGRADE_MATRIX=true
- fi
-
- java -version
- cd ~/cassandra-dtest
- mkdir -p /tmp/dtest
-
- echo "env: $(env)"
- echo "** done env"
- mkdir -p /tmp/results/dtests
- # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
- export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt`
- set -o pipefail && cd ~/cassandra-dtest && pytest --execute-upgrade-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_upgradetests_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
- - store_test_results:
- path: /tmp/results
- - store_artifacts:
- path: /tmp/dtest
- destination: dtest_j8_upgradetests_without_vnodes
- - store_artifacts:
- path: ~/cassandra-dtest/logs
- destination: dtest_j8_upgradetests_without_vnodes_logs
+ - persist_to_workspace:
+ root: /home/cassandra
+ paths:
+ - dtest_jars
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
@@ -606,6 +676,16 @@ workflows:
build_and_run_tests:
jobs:
- build
+ - start_jvm_upgrade_dtest:
+ type: approval
+ requires:
+ - build
+ - dtest_jars_build:
+ requires:
+ - start_jvm_upgrade_dtest
+ - j8_jvm_upgrade_dtests:
+ requires:
+ - dtest_jars_build
- j8_unit_tests:
requires:
- build
@@ -643,453 +723,3 @@ workflows:
- j8_upgradetests-no-vnodes:
requires:
- start_upgrade_tests
-
-# Original config.yml file:
-# version: 2.1
-#
-# default_env_vars: &default_env_vars
-# JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
-# ANT_HOME: /usr/share/ant
-# LANG: en_US.UTF-8
-# KEEP_TEST_DIR: true
-# DEFAULT_DIR: /home/cassandra/cassandra-dtest
-# PYTHONIOENCODING: utf-8
-# PYTHONUNBUFFERED: true
-# CASS_DRIVER_NO_EXTENSIONS: true
-# CASS_DRIVER_NO_CYTHON: true
-# #Skip all syncing to disk to avoid performance issues in flaky CI environments
-# CASSANDRA_SKIP_SYNC: true
-# DTEST_REPO: git://github.com/apache/cassandra-dtest.git
-# DTEST_BRANCH: master
-# CCM_MAX_HEAP_SIZE: 1024M
-# CCM_HEAP_NEWSIZE: 256M
-#
-# j8_par_executor: &j8_par_executor
-# executor:
-# name: java8-executor
-# #exec_resource_class: xlarge
-# parallelism: 4
-#
-# j8_seq_executor: &j8_seq_executor
-# executor:
-# name: java8-executor
-# #exec_resource_class: xlarge
-# parallelism: 1 # sequential, single container tests: no parallelism benefits
-#
-# with_dtests_jobs: &with_dtest_jobs
-# jobs:
-# - build
-# # Java 8 unit tests will be run automatically
-# - j8_unit_tests:
-# requires:
-# - build
-# - j8_jvm_dtests:
-# requires:
-# - build
-# # specialized unit tests (all run on request using Java 8)
-# - start_utests_long:
-# type: approval
-# requires:
-# - build
-# - utests_long:
-# requires:
-# - start_utests_long
-# - start_utests_compression:
-# type: approval
-# requires:
-# - build
-# - utests_compression:
-# requires:
-# - start_utests_compression
-# # Java 8 dtests (on request)
-# - start_j8_dtests:
-# type: approval
-# requires:
-# - build
-# - j8_dtests-with-vnodes:
-# requires:
-# - start_j8_dtests
-# - j8_dtests-no-vnodes:
-# requires:
-# - start_j8_dtests
-# # Java 8 upgrade tests
-# - start_upgrade_tests:
-# type: approval
-# requires:
-# - build
-# - j8_upgradetests-no-vnodes:
-# requires:
-# - start_upgrade_tests
-#
-# with_dtest_jobs_only: &with_dtest_jobs_only
-# jobs:
-# - build
-# - j8_dtests-with-vnodes:
-# requires:
-# - build
-# - j8_dtests-no-vnodes:
-# requires:
-# - build
-#
-# workflows:
-# version: 2
-# build_and_run_tests: *with_dtest_jobs
-# #build_and_run_tests: *with_dtest_jobs_only
-#
-# executors:
-# java8-executor:
-# parameters:
-# exec_resource_class:
-# type: string
-# default: medium
-# docker:
-# - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
-# resource_class: << parameters.exec_resource_class >>
-# working_directory: ~/
-# shell: /bin/bash -eo pipefail -l
-# environment:
-# <<: *default_env_vars
-# JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
-# JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
-#
-# jobs:
-# build:
-# executor: java8-executor
-# parallelism: 1 # This job doesn't benefit from parallelism
-# steps:
-# - log_environment
-# - clone_cassandra
-# - build_cassandra
-# - run_eclipse_warnings
-# - persist_to_workspace:
-# root: /home/cassandra
-# paths:
-# - cassandra
-# - .m2
-#
-# j8_unit_tests:
-# <<: *j8_par_executor
-# steps:
-# - attach_workspace:
-# at: /home/cassandra
-# - create_junit_containers
-# - log_environment
-# - run_parallel_junit_tests
-#
-# j8_jvm_dtests:
-# <<: *j8_seq_executor
-# steps:
-# - attach_workspace:
-# at: /home/cassandra
-# - run_junit_tests:
-# target: test-jvm-dtest-forking
-#
-# utests_long:
-# <<: *j8_seq_executor
-# steps:
-# - attach_workspace:
-# at: /home/cassandra
-# - run_junit_tests:
-# target: long-test
-#
-# utests_compression:
-# <<: *j8_par_executor
-# steps:
-# - attach_workspace:
-# at: /home/cassandra
-# - create_junit_containers
-# - log_environment
-# - run_parallel_junit_tests:
-# target: testclasslist-compression
-#
-# j8_dtests-with-vnodes:
-# <<: *j8_par_executor
-# steps:
-# - attach_workspace:
-# at: /home/cassandra
-# - clone_dtest
-# - create_venv
-# - create_dtest_containers:
-# file_tag: j8_with_vnodes
-# run_dtests_extra_args: '--use-vnodes --skip-resource-intensive-tests'
-# - run_dtests:
-# file_tag: j8_with_vnodes
-# pytest_extra_args: '--use-vnodes --num-tokens=32 --skip-resource-intensive-tests'
-#
-# j8_dtests-no-vnodes:
-# <<: *j8_par_executor
-# steps:
-# - attach_workspace:
-# at: /home/cassandra
-# - clone_dtest
-# - create_venv
-# - create_dtest_containers:
-# file_tag: j8_without_vnodes
-# run_dtests_extra_args: '--skip-resource-intensive-tests'
-# - run_dtests:
-# file_tag: j8_without_vnodes
-# pytest_extra_args: '--skip-resource-intensive-tests'
-#
-# j8_upgradetests-no-vnodes:
-# <<: *j8_par_executor
-# steps:
-# - attach_workspace:
-# at: /home/cassandra
-# - clone_dtest
-# - create_venv
-# - create_dtest_containers:
-# file_tag: j8_upgradetests_without_vnodes
-# run_dtests_extra_args: '--execute-upgrade-tests'
-# extra_env_args: 'RUN_STATIC_UPGRADE_MATRIX=true'
-# tests_filter_pattern: '^upgrade_tests'
-# - run_dtests:
-# file_tag: j8_upgradetests_without_vnodes
-# extra_env_args: 'RUN_STATIC_UPGRADE_MATRIX=true'
-# pytest_extra_args: '--execute-upgrade-tests'
-#
-# commands:
-# log_environment:
-# steps:
-# - run:
-# name: Log Environment Information
-# command: |
-# echo '*** id ***'
-# id
-# echo '*** cat /proc/cpuinfo ***'
-# cat /proc/cpuinfo
-# echo '*** free -m ***'
-# free -m
-# echo '*** df -m ***'
-# df -m
-# echo '*** ifconfig -a ***'
-# ifconfig -a
-# echo '*** uname -a ***'
-# uname -a
-# echo '*** mount ***'
-# mount
-# echo '*** env ***'
-# env
-# echo '*** java ***'
-# which java
-# java -version
-#
-# clone_cassandra:
-# steps:
-# - run:
-# name: Clone Cassandra Repository (via git)
-# command: |
-# git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH git://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra
-#
-# clone_dtest:
-# steps:
-# - run:
-# name: Clone Cassandra dtest Repository (via git)
-# command: |
-# git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
-#
-# build_cassandra:
-# steps:
-# - run:
-# name: Build Cassandra
-# command: |
-# export PATH=$JAVA_HOME/bin:$PATH
-# cd ~/cassandra
-# # Loop to prevent failure due to maven-ant-tasks not downloading a jar..
-# for x in $(seq 1 3); do
-# ${ANT_HOME}/bin/ant clean jar
-# RETURN=\"$?\"
-# if [ \"${RETURN}\" -eq \"0\" ]; then
-# break
-# fi
-# done
-# # Exit, if we didn't build successfully
-# if [ \"${RETURN}\" -ne \"0\" ]; then
-# echo \"Build failed with exit code: ${RETURN}\"
-# exit ${RETURN}
-# fi
-# no_output_timeout: 15m
-#
-# run_eclipse_warnings:
-# steps:
-# - run:
-# name: Run eclipse-warnings
-# command: |
-# export PATH=$JAVA_HOME/bin:$PATH
-# cd ~/cassandra
-# ant eclipse-warnings
-#
-# create_junit_containers:
-# steps:
-# - run:
-# name: Determine Unit Tests to Run
-# command: |
-# # reminder: this code (along with all the steps) is independently executed on every circle container
-# # so the goal here is to get the circleci script to return the tests *this* container will run
-# # which we do via the `circleci` cli tool.
-#
-# rm -fr ~/cassandra-dtest/upgrade_tests
-# echo \"***java tests***\"
-#
-# # get all of our unit test filenames
-# set -eo pipefail && circleci tests glob \"$HOME/cassandra/test/unit/**/*.java\" > /tmp/all_java_unit_tests.txt
-#
-# # split up the unit tests into groups based on the number of containers we have
-# set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
-# set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | cut -c 37-1000000 | grep \"Test\\.java$\" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
-# echo \"** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt\"
-# cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
-#
-# no_output_timeout: 15m
-#
-# run_junit_tests:
-# parameters:
-# target:
-# type: string
-# no_output_timeout:
-# type: string
-# default: 15m
-# steps:
-# - run:
-# name: Run Unit Tests (<<parameters.target>>)
-# # Please note that we run `clean` and therefore rebuild the project, as we can't run tests on Java 8 in case
-# # based on Java 11 builds.
-# command: |
-# export PATH=$JAVA_HOME/bin:$PATH
-# time mv ~/cassandra /tmp
-# cd /tmp/cassandra
-# ant clean <<parameters.target>>
-# no_output_timeout: <<parameters.no_output_timeout>>
-# - store_test_results:
-# path: /tmp/cassandra/build/test/output/
-# - store_artifacts:
-# path: /tmp/cassandra/build/test/output
-# destination: junitxml
-# - store_artifacts:
-# path: /tmp/cassandra/build/test/logs
-# destination: logs
-#
-# run_parallel_junit_tests:
-# parameters:
-# target:
-# type: string
-# default: testclasslist
-# no_output_timeout:
-# type: string
-# default: 15m
-# steps:
-# - run:
-# name: Run Unit Tests (<<parameters.target>>)
-# # Please note that we run `clean` and therefore rebuild the project, as we can't run tests on Java 8 in case
-# # based on Java 11 builds.
-# command: |
-# export PATH=$JAVA_HOME/bin:$PATH
-# time mv ~/cassandra /tmp
-# cd /tmp/cassandra
-# ant <<parameters.target>> -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
-# no_output_timeout: <<parameters.no_output_timeout>>
-# - store_test_results:
-# path: /tmp/cassandra/build/test/output/
-# - store_artifacts:
-# path: /tmp/cassandra/build/test/output
-# destination: junitxml
-# - store_artifacts:
-# path: /tmp/cassandra/build/test/logs
-# destination: logs
-#
-# create_venv:
-# steps:
-# - run:
-# name: Configure virtualenv and python Dependencies
-# command: |
-# # note, this should be super quick as all dependencies should be pre-installed in the docker image
-# # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
-# # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
-# # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
-# source ~/env/bin/activate
-# export PATH=$JAVA_HOME/bin:$PATH
-# pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
-# pip3 freeze
-#
-# create_dtest_containers:
-# parameters:
-# file_tag:
-# type: string
-# run_dtests_extra_args:
-# type: string
-# default: ''
-# extra_env_args:
-# type: string
-# default: ''
-# tests_filter_pattern:
-# type: string
-# default: ''
-# steps:
-# - run:
-# name: Determine Tests to Run (<<parameters.file_tag>>)
-# no_output_timeout: 5m
-# command: |
-# # reminder: this code (along with all the steps) is independently executed on every circle container
-# # so the goal here is to get the circleci script to return the tests *this* container will run
-# # which we do via the `circleci` cli tool.
-#
-# cd cassandra-dtest
-# source ~/env/bin/activate
-# export PATH=$JAVA_HOME/bin:$PATH
-#
-# if [ -n '<<parameters.extra_env_args>>' ]; then
-# export <<parameters.extra_env_args>>
-# fi
-#
-# echo \"***Collected DTests (<<parameters.file_tag>>)***\"
-# set -eo pipefail && ./run_dtests.py <<parameters.run_dtests_extra_args>> --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_<<parameters.file_tag>>_raw --cassandra-dir=../cassandra
-# if [ -z '<<parameters.tests_filter_pattern>>' ]; then
-# mv /tmp/all_dtest_tests_<<parameters.file_tag>>_raw /tmp/all_dtest_tests_<<parameters.file_tag>>
-# else
-# grep -e '<<parameters.tests_filter_pattern>>' /tmp/all_dtest_tests_<<parameters.file_tag>>_raw > /tmp/all_dtest_tests_<<parameters.file_tag>> || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }
-# fi
-# set -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_<<parameters.file_tag>> > /tmp/split_dtest_tests_<<parameters.file_tag>>.txt
-# cat /tmp/split_dtest_tests_<<parameters.file_tag>>.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_<<parameters.file_tag>>_final.txt
-# cat /tmp/split_dtest_tests_<<parameters.file_tag>>_final.txt
-#
-# run_dtests:
-# parameters:
-# file_tag:
-# type: string
-# pytest_extra_args:
-# type: string
-# default: ''
-# extra_env_args:
-# type: string
-# default: ''
-# steps:
-# - run:
-# name: Run dtests (<<parameters.file_tag>>)
-# no_output_timeout: 15m
-# command: |
-# echo \"cat /tmp/split_dtest_tests_<<parameters.file_tag>>_final.txt\"
-# cat /tmp/split_dtest_tests_<<parameters.file_tag>>_final.txt
-#
-# source ~/env/bin/activate
-# export PATH=$JAVA_HOME/bin:$PATH
-# if [ -n '<<parameters.extra_env_args>>' ]; then
-# export <<parameters.extra_env_args>>
-# fi
-#
-# java -version
-# cd ~/cassandra-dtest
-# mkdir -p /tmp/dtest
-#
-# echo \"env: $(env)\"
-# echo \"** done env\"
-# mkdir -p /tmp/results/dtests
-# # we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
-# export SPLIT_TESTS=`cat /tmp/split_dtest_tests_<<parameters.file_tag>>_final.txt`
-# set -o pipefail && cd ~/cassandra-dtest && pytest <<parameters.pytest_extra_args>> --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_<<parameters.file_tag>>.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
-# - store_test_results:
-# path: /tmp/results
-# - store_artifacts:
-# path: /tmp/dtest
-# destination: dtest_<<parameters.file_tag>>
-# - store_artifacts:
-# path: ~/cassandra-dtest/logs
-# destination: dtest_<<parameters.file_tag>>_logs
\ No newline at end of file
diff --git a/CHANGES.txt b/CHANGES.txt
index 2edc399..025e2ae 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
3.0.20
+ * Run in-jvm upgrade dtests in circleci (CASSANDRA-15506)
* Include updates to static column in mutation size calculations (CASSANDRA-15293)
* Fix point-in-time recoevery ignoring timestamp of updates to static columns (CASSANDRA-15292)
* GC logs are also put under $CASSANDRA_LOG_DIR (CASSANDRA-14306)
diff --git a/build.xml b/build.xml
index 5d86a3f..d24f745 100644
--- a/build.xml
+++ b/build.xml
@@ -66,6 +66,8 @@
<property name="test.microbench.src" value="${test.dir}/microbench"/>
<property name="test.distributed.src" value="${test.dir}/distributed"/>
<property name="test.distributed.listfile" value="ant-jvm-dtest-list"/>
+ <property name="test.distributed.upgrade.listfile" value="ant-jvm-dtest-upgrade-list"/>
+ <property name="test.distributed.upgrade.package" value="org.apache.cassandra.distributed.upgrade"/>
<property name="dist.dir" value="${build.dir}/dist"/>
<property name="tmp.dir" value="${java.io.tmpdir}"/>
@@ -1810,6 +1812,39 @@
</testmacro>
</target>
+ <!-- In-JVM upgrade dtests -->
+ <target name="list-jvm-upgrade-dtests" depends="build-test">
+ <java classname="org.apache.cassandra.distributed.test.TestLocator" fork="no">
+ <classpath>
+ <path refid="cassandra.classpath" />
+ <pathelement location="${test.classes}"/>
+ <pathelement location="${test.conf}"/>
+ <fileset dir="${test.lib}">
+ <include name="**/*.jar" />
+ </fileset>
+ </classpath>
+ <arg value="${test.distributed.upgrade.listfile}"/>
+ <arg value="${test.distributed.upgrade.package}"/>
+ </java>
+ </target>
+
+ <target name="test-jvm-upgrade-dtest-forking" depends="list-jvm-upgrade-dtests" description="Execute In-JVM 'distributed' upgrade tests" >
+ <chmod file="${test.distributed.upgrade.listfile}" perm="+x"/>
+ <exec executable="./${test.distributed.upgrade.listfile}" failonerror="true"/>
+ <delete file="${test.distributed.upgrade.listfile}"/>
+ </target>
+
+ <target name="test-jvm-upgrade-dtest" depends="build-test" description="Execute in-jvm dtests">
+ <testmacro inputdir="${test.distributed.src}" timeout="${test.distributed.timeout}" forkmode="once" showoutput="true" filter="**/upgrade/*Test.java">
+ <jvmarg value="-Dlogback.configurationFile=test/conf/logback-dtest.xml"/>
+ <jvmarg value="-Dcassandra.ring_delay_ms=10000"/>
+ <jvmarg value="-Dcassandra.tolerate_sstable_size=true"/>
+ <jvmarg value="-Djava.io.tmpdir=${tmp.dir}"/>
+ <jvmarg value="-Dcassandra.skip_sync=true" />
+ <jvmarg value="-XX:MaxMetaspaceSize=512M"/>
+ </testmacro>
+ </target>
+
<!-- Use this with an FQDN for test class, and a csv list of methods like this:
ant test-jvm-dtest-some -Dtest.name=org.apache.cassandra.distributed.test.ResourceLeakTest -Dtest.methods=looperTest
-->
diff --git a/test/distributed/org/apache/cassandra/distributed/test/TestLocator.java b/test/distributed/org/apache/cassandra/distributed/test/TestLocator.java
index a7ad400..e566910 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/TestLocator.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/TestLocator.java
@@ -39,14 +39,18 @@ public class TestLocator
public static void main(String[] args) throws Throwable
{
String outputFileName = defaultOutputFileName;
- if (args.length == 1)
+ if (args.length >= 1)
{
outputFileName = args[0];
}
+ String testPackage = TestLocator.testPackage;
+ if (args.length == 2)
+ testPackage = args[1];
try (FileWriter fileWriter = new FileWriter(outputFileName);
PrintWriter printWriter = new PrintWriter(fileWriter))
{
printWriter.println("#!/bin/bash");
+ printWriter.println("ret=0");
for (Class testClass : locateClasses(testPackage))
{
for (Method method : testClass.getMethods())
@@ -57,8 +61,10 @@ public class TestLocator
printWriter.println(String.format(testCommandFormat,
testClass.getName(),
method.getName()));
+ printWriter.println("if [ $? -ne 0 ]; then ret=1; fi");
}
}
+ printWriter.println("exit $ret");
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org