You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by ma...@apache.org on 2020/01/29 08:45:51 UTC
[cassandra] branch trunk updated (9299e67 -> ec4174d)
This is an automated email from the ASF dual-hosted git repository.
marcuse pushed a change to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra.git.
from 9299e67 Merge branch 'cassandra-3.11' into trunk
new 6e3bf54 Use testclasslist for jvm dtests in circleci
new a6dd3b8 Merge branch 'cassandra-2.2' into cassandra-3.0
new 3bcabdc Merge branch 'cassandra-3.0' into cassandra-3.11
new ec4174d Merge branch 'cassandra-3.11' into trunk
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.circleci/config-2_1.yml | 86 +++++++++++-----
.circleci/config-2_1.yml.high_res.patch | 18 +++-
.circleci/config.yml | 166 ++++++++++++++++++++++++++++---
.circleci/config.yml.HIGHRES | 169 ++++++++++++++++++++++++++++----
.circleci/config.yml.LOWRES | 163 +++++++++++++++++++++++++++---
5 files changed, 524 insertions(+), 78 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org
[cassandra] 01/01: Merge branch 'cassandra-3.11' into trunk
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
marcuse pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra.git
commit ec4174d91ff50e7d9786282eebd9d7d45109287e
Merge: 9299e67 3bcabdc
Author: Marcus Eriksson <ma...@apache.org>
AuthorDate: Wed Jan 29 09:38:01 2020 +0100
Merge branch 'cassandra-3.11' into trunk
.circleci/config-2_1.yml | 86 +++++++++++-----
.circleci/config-2_1.yml.high_res.patch | 18 +++-
.circleci/config.yml | 166 ++++++++++++++++++++++++++++---
.circleci/config.yml.HIGHRES | 169 ++++++++++++++++++++++++++++----
.circleci/config.yml.LOWRES | 163 +++++++++++++++++++++++++++---
5 files changed, 524 insertions(+), 78 deletions(-)
diff --cc .circleci/config-2_1.yml
index 99a0390,5cbe288..650b9a0
--- a/.circleci/config-2_1.yml
+++ b/.circleci/config-2_1.yml
@@@ -28,151 -41,76 +40,150 @@@ j8_seq_executor: &j8_seq_executo
#exec_resource_class: xlarge
parallelism: 1 # sequential, single container tests: no parallelism benefits
-with_dtests_jobs: &with_dtest_jobs
- jobs:
- - build
- # Java 8 unit tests will be run automatically
- - j8_unit_tests:
- requires:
- - build
- - j8_jvm_dtests:
- requires:
- - build
- # specialized unit tests (all run on request using Java 8)
- - start_utests_long:
- type: approval
- requires:
- - build
- - utests_long:
- requires:
- - start_utests_long
- - start_utests_compression:
- type: approval
- requires:
- - build
- - utests_compression:
- requires:
- - start_utests_compression
- - start_utests_stress:
- type: approval
- requires:
- - build
- - utests_stress:
- requires:
- - start_utests_stress
- - start_jvm_upgrade_dtest:
- type: approval
- - j8_dtest_jars_build:
- requires:
- - build
- - start_jvm_upgrade_dtest
- - j8_jvm_upgrade_dtests:
- requires:
- - j8_dtest_jars_build
- # Java 8 dtests (on request)
- - start_j8_dtests:
- type: approval
- requires:
- - build
- - j8_dtests-with-vnodes:
- requires:
- - start_j8_dtests
- - j8_dtests-no-vnodes:
- requires:
- - start_j8_dtests
- # Java 8 upgrade tests
- - start_upgrade_tests:
- type: approval
- requires:
- - build
- - j8_upgradetests-no-vnodes:
- requires:
- - start_upgrade_tests
+j11_par_executor: &j11_par_executor
+ executor:
+ name: java11-executor
+ #exec_resource_class: xlarge
+ parallelism: 4
+
- j11_seq_executor: &j11_seq_executor
++j11_small_par_executor: &j11_small_par_executor
+ executor:
+ name: java11-executor
+ #exec_resource_class: xlarge
- parallelism: 1 # sequential, single container tests: no parallelism benefits
++ parallelism: 1
-with_dtest_jobs_only: &with_dtest_jobs_only
+j8_with_dtests_jobs: &j8_with_dtests_jobs
+ jobs:
+ - j8_build
+ # Java 8 unit tests will be run automatically
+ - j8_unit_tests:
+ requires:
+ - j8_build
+ - j8_jvm_dtests:
+ requires:
+ - j8_build
+ # Java 11 unit tests (on request, currently not working)
+ - start_j11_unit_tests:
+ type: approval
+ requires:
+ - j8_build
+ - j11_unit_tests:
+ requires:
+ - start_j11_unit_tests
+ # specialized unit tests (all run on request using Java 8)
+ - start_utests_long:
+ type: approval
+ requires:
+ - j8_build
+ - utests_long:
+ requires:
+ - start_utests_long
+ - start_utests_compression:
+ type: approval
+ requires:
+ - j8_build
+ - utests_compression:
+ requires:
+ - start_utests_compression
+ - start_utests_stress:
+ type: approval
+ requires:
+ - j8_build
+ - utests_stress:
+ requires:
+ - start_utests_stress
+ - start_utests_fqltool:
+ type: approval
+ requires:
+ - j8_build
+ - utests_fqltool:
+ requires:
+ - start_utests_fqltool
+ - start_jvm_upgrade_dtest:
+ type: approval
++ - j8_dtest_jars_build:
+ requires:
+ - j8_build
- - dtest_jars_build:
- requires:
+ - start_jvm_upgrade_dtest
+ - j8_jvm_upgrade_dtests:
+ requires:
- - dtest_jars_build
++ - j8_dtest_jars_build
+ # Java 8 dtests (on request)
+ - start_j8_dtests:
+ type: approval
+ requires:
+ - j8_build
+ - j8_dtests-with-vnodes:
+ requires:
+ - start_j8_dtests
+ - j8_dtests-no-vnodes:
+ requires:
+ - start_j8_dtests
+ # Java 11 dtests (on request)
+ - start_j11_dtests:
+ type: approval
+ requires:
+ - j8_build
+ - j11_dtests-with-vnodes:
+ requires:
+ - start_j11_dtests
+ - j11_dtests-no-vnodes:
+ requires:
+ - start_j11_dtests
+ # Java 8 upgrade tests
+ - start_upgrade_tests:
+ type: approval
+ requires:
+ - j8_build
+ - j8_upgradetests-no-vnodes:
+ requires:
+ - start_upgrade_tests
+
+j11_with_dtests_jobs: &j11_with_dtests_jobs
+ jobs:
+ - j11_build
+ # Java 11 unit tests (on request, currently not working)
+ - start_j11_unit_tests:
+ type: approval
+ requires:
+ - j11_build
+ - j11_unit_tests:
+ requires:
+ - start_j11_unit_tests
+ - j11_jvm_dtests:
+ requires:
+ - j11_build
+ # Java 11 dtests (on request)
+ - start_j11_dtests:
+ type: approval
+ requires:
+ - j11_build
+ - j11_dtests-with-vnodes:
+ requires:
+ - start_j11_dtests
+ - j11_dtests-no-vnodes:
+ requires:
+ - start_j11_dtests
+
+j8_with_dtest_jobs_only: &j8_with_dtest_jobs_only
jobs:
- - build
+ - j8_build
- j8_dtests-with-vnodes:
requires:
- - build
+ - j8_build
- j8_dtests-no-vnodes:
requires:
- - build
+ - j8_build
+
+j11_with_dtest_jobs_only: &j11_with_dtest_jobs_only
+ jobs:
+ - build
+ - j11-with-vnodes:
+ requires:
+ - j11_build
+ - j11_dtests-no-vnodes:
+ requires:
+ - j11_build
workflows:
version: 2
@@@ -197,45 -133,22 +208,45 @@@ executors
JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ java11-executor:
+ parameters:
+ exec_resource_class:
+ type: string
+ default: medium
+ docker:
+ - image: spod/cassandra-testing-ubuntu1810-java11:20181210
+ resource_class: << parameters.exec_resource_class >>
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ environment:
+ <<: *default_env_vars
+ JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ CASSANDRA_USE_JDK11: true
+
+build_common: &build_common
+ parallelism: 1 # This job doesn't benefit from parallelism
+ steps:
+ - log_environment
+ - clone_cassandra
+ - build_cassandra
+ - run_eclipse_warnings
+ - persist_to_workspace:
+ root: /home/cassandra
+ paths:
+ - cassandra
+ - .m2
+
jobs:
- build:
+ j8_build:
executor: java8-executor
- parallelism: 1 # This job doesn't benefit from parallelism
- steps:
- - log_environment
- - clone_cassandra
- - build_cassandra
- - run_eclipse_warnings
- - persist_to_workspace:
- root: /home/cassandra
- paths:
- - cassandra
- - .m2
+ <<: *build_common
+
+ j11_build:
+ executor: java11-executor
+ <<: *build_common
- dtest_jars_build:
+ j8_dtest_jars_build:
executor: java8-executor
parallelism: 1
steps:
@@@ -261,34 -174,25 +272,46 @@@
steps:
- attach_workspace:
at: /home/cassandra
- - run_junit_tests:
- target: test-jvm-dtest-forking
+ - create_junit_containers:
+ classlistprefix: distributed
+ extra_filters: "| grep -v upgrade"
+ - log_environment
+ - run_parallel_junit_tests:
+ classlistprefix: distributed
- j11_unit_tests:
- <<: *j11_par_executor
++ j11_jvm_dtests:
++ <<: *j11_small_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
- - create_junit_containers
++ - create_junit_containers:
++ classlistprefix: distributed
++ extra_filters: "| grep -v upgrade"
+ - log_environment
- - run_parallel_junit_tests
++ - run_parallel_junit_tests:
++ classlistprefix: distributed
+
- j11_jvm_dtests:
- <<: *j11_seq_executor
+ j8_jvm_upgrade_dtests:
+ <<: *j8_medium_par_executor
steps:
- attach_workspace:
at: /home/cassandra
- - run_junit_tests:
- target: test-jvm-dtest-forking
+ - create_junit_containers:
+ classlistprefix: distributed
+ extra_filters: "| grep upgrade"
+ - log_environment
+ - run_parallel_junit_tests:
+ classlistprefix: distributed
- j8_jvm_upgrade_dtests:
- <<: *j8_seq_executor
++ j11_unit_tests:
++ <<: *j11_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
- - run_junit_tests:
- target: test-jvm-upgrade-dtest-forking
++ - create_junit_containers
++ - log_environment
++ - run_parallel_junit_tests
+
utests_long:
<<: *j8_seq_executor
steps:
diff --cc .circleci/config-2_1.yml.high_res.patch
index 4d5e9e1,3c85668..8f9132c
--- a/.circleci/config-2_1.yml.high_res.patch
+++ b/.circleci/config-2_1.yml.high_res.patch
@@@ -4,17 -10,19 +4,31 @@@
---
> exec_resource_class: xlarge
> parallelism: 100
- 28c28
-29,30c29,30
++28,29c28,29
< #exec_resource_class: xlarge
+ < parallelism: 1
---
> exec_resource_class: xlarge
+ > parallelism: 2
-35,36c35,36
+34,35c34,35
< #exec_resource_class: xlarge
+ < parallelism: 1
+ ---
+ > exec_resource_class: xlarge
+ > parallelism: 2
-41c41
++40c40
++< #exec_resource_class: xlarge
++---
++> exec_resource_class: xlarge
++46,47c46,47
+ < #exec_resource_class: xlarge
+< parallelism: 4
---
> exec_resource_class: xlarge
+> parallelism: 100
- 40c40
++52,53c52,53
+< #exec_resource_class: xlarge
++< parallelism: 1
+---
+> exec_resource_class: xlarge
++> parallelism: 2
diff --cc .circleci/config.yml
index d23d344,4108359..6e07fd2
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@@ -51,29 -93,8 +92,29 @@@ jobs
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
- parallelism: 1
+ parallelism: 4
steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
- name: Determine Unit Tests to Run
++ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
- set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | cut -c 37-1000000 | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
++ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
- run:
name: Log Environment Information
command: |
@@@ -97,102 -118,41 +138,105 @@@
which java
java -version
- run:
- name: Clone Cassandra Repository (via git)
- command: |
- git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH git://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra
- - run:
- name: Build Cassandra
+ name: Run Unit Tests (testclasslist)
command: |
export PATH=$JAVA_HOME/bin:$PATH
- cd ~/cassandra
- # Loop to prevent failure due to maven-ant-tasks not downloading a jar..
- for x in $(seq 1 3); do
- ${ANT_HOME}/bin/ant clean jar
- RETURN="$?"
- if [ "${RETURN}" -eq "0" ]; then
- break
- fi
- done
- # Exit, if we didn't build successfully
- if [ "${RETURN}" -ne "0" ]; then
- echo "Build failed with exit code: ${RETURN}"
- exit ${RETURN}
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
++ if [ -d ~/dtest_jars ]; then
++ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ ant testclasslist -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_dtests-with-vnodes:
+ docker:
+ - image: spod/cassandra-testing-ubuntu1810-java11:20181210
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
- run:
- name: Run eclipse-warnings
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
- cd ~/cassandra
- ant eclipse-warnings
- - persist_to_workspace:
- root: /home/cassandra
- paths:
- - cassandra
- - .m2
+ pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-re [...]
+ - run:
+ name: Run dtests (j11_with_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the ex [...]
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_with_vnodes_logs
environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
@@@ -492,130 -458,6 +539,174 @@@
- CCM_HEAP_NEWSIZE: 256M
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j11_jvm_dtests:
+ docker:
+ - image: spod/cassandra-testing-ubuntu1810-java11:20181210
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
- name: Run Unit Tests (test-jvm-dtest-forking)
++ name: Determine distributed Tests to Run
++ command: |
++ # reminder: this code (along with all the steps) is independently executed on every circle container
++ # so the goal here is to get the circleci script to return the tests *this* container will run
++ # which we do via the `circleci` cli tool.
++
++ rm -fr ~/cassandra-dtest/upgrade_tests
++ echo "***java tests***"
++
++ # get all of our unit test filenames
++ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt
++
++ # split up the unit tests into groups based on the number of containers we have
++ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
++ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep -v upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
++ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
++ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
++ no_output_timeout: 15m
++ - run:
++ name: Log Environment Information
++ command: |
++ echo '*** id ***'
++ id
++ echo '*** cat /proc/cpuinfo ***'
++ cat /proc/cpuinfo
++ echo '*** free -m ***'
++ free -m
++ echo '*** df -m ***'
++ df -m
++ echo '*** ifconfig -a ***'
++ ifconfig -a
++ echo '*** uname -a ***'
++ uname -a
++ echo '*** mount ***'
++ mount
++ echo '*** env ***'
++ env
++ echo '*** java ***'
++ which java
++ java -version
++ - run:
++ name: Run Unit Tests (testclasslist)
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
- ant test-jvm-dtest-forking
++ if [ -d ~/dtest_jars ]; then
++ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
++ fi
++ ant testclasslist -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_build:
+ docker:
+ - image: spod/cassandra-testing-ubuntu1810-java11:20181210
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Clone Cassandra Repository (via git)
+ command: |
+ git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH git://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra
+ - run:
+ name: Build Cassandra
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ cd ~/cassandra
+ # Loop to prevent failure due to maven-ant-tasks not downloading a jar..
+ for x in $(seq 1 3); do
+ ${ANT_HOME}/bin/ant clean realclean jar
+ RETURN="$?"
+ if [ "${RETURN}" -eq "0" ]; then
+ break
+ fi
+ done
+ # Exit, if we didn't build successfully
+ if [ "${RETURN}" -ne "0" ]; then
+ echo "Build failed with exit code: ${RETURN}"
+ exit ${RETURN}
+ fi
+ no_output_timeout: 15m
+ - run:
+ name: Run eclipse-warnings
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ cd ~/cassandra
+ ant eclipse-warnings
+ - persist_to_workspace:
+ root: /home/cassandra
+ paths:
+ - cassandra
+ - .m2
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
j8_dtests-with-vnodes:
docker:
- image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
@@@ -1158,19 -837,11 +1293,18 @@@ workflows
- utests_stress:
requires:
- start_utests_stress
+ - start_utests_fqltool:
+ type: approval
+ requires:
+ - j8_build
+ - utests_fqltool:
+ requires:
+ - start_utests_fqltool
- start_jvm_upgrade_dtest:
type: approval
+ - j8_dtest_jars_build:
requires:
- - build
+ - j8_build
- - dtest_jars_build:
- requires:
- start_jvm_upgrade_dtest
- j8_jvm_upgrade_dtests:
requires:
diff --cc .circleci/config.yml.HIGHRES
index 4bf09f0,ac61b7f..47350fd
--- a/.circleci/config.yml.HIGHRES
+++ b/.circleci/config.yml.HIGHRES
@@@ -41,39 -83,18 +82,39 @@@ jobs
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: git://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: master
- - CCM_MAX_HEAP_SIZE: 2048M
- - CCM_HEAP_NEWSIZE: 512M
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- build:
+ j11_unit_tests:
docker:
- - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
- resource_class: medium
+ - image: spod/cassandra-testing-ubuntu1810-java11:20181210
+ resource_class: xlarge
working_directory: ~/
shell: /bin/bash -eo pipefail -l
- parallelism: 1
+ parallelism: 100
steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
- name: Determine Unit Tests to Run
++ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
- set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | cut -c 37-1000000 | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
++ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
- run:
name: Log Environment Information
command: |
@@@ -97,102 -118,41 +138,105 @@@
which java
java -version
- run:
- name: Clone Cassandra Repository (via git)
- command: |
- git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH git://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra
- - run:
- name: Build Cassandra
+ name: Run Unit Tests (testclasslist)
command: |
export PATH=$JAVA_HOME/bin:$PATH
- cd ~/cassandra
- # Loop to prevent failure due to maven-ant-tasks not downloading a jar..
- for x in $(seq 1 3); do
- ${ANT_HOME}/bin/ant clean jar
- RETURN="$?"
- if [ "${RETURN}" -eq "0" ]; then
- break
- fi
- done
- # Exit, if we didn't build successfully
- if [ "${RETURN}" -ne "0" ]; then
- echo "Build failed with exit code: ${RETURN}"
- exit ${RETURN}
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
++ if [ -d ~/dtest_jars ]; then
++ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ ant testclasslist -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_dtests-with-vnodes:
+ docker:
+ - image: spod/cassandra-testing-ubuntu1810-java11:20181210
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
- run:
- name: Run eclipse-warnings
+ name: Log Environment Information
command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
- cd ~/cassandra
- ant eclipse-warnings
- - persist_to_workspace:
- root: /home/cassandra
- paths:
- - cassandra
- - .m2
+ pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-re [...]
+ - run:
+ name: Run dtests (j11_with_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the ex [...]
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_with_vnodes_logs
environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
@@@ -488,137 -454,10 +535,178 @@@
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: git://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: master
- - CCM_MAX_HEAP_SIZE: 2048M
- - CCM_HEAP_NEWSIZE: 512M
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j11_jvm_dtests:
+ docker:
+ - image: spod/cassandra-testing-ubuntu1810-java11:20181210
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
- parallelism: 1
++ parallelism: 2
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
- name: Run Unit Tests (test-jvm-dtest-forking)
++ name: Determine distributed Tests to Run
++ command: |
++ # reminder: this code (along with all the steps) is independently executed on every circle container
++ # so the goal here is to get the circleci script to return the tests *this* container will run
++ # which we do via the `circleci` cli tool.
++
++ rm -fr ~/cassandra-dtest/upgrade_tests
++ echo "***java tests***"
++
++ # get all of our unit test filenames
++ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt
++
++ # split up the unit tests into groups based on the number of containers we have
++ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
++ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep -v upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
++ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
++ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
++ no_output_timeout: 15m
++ - run:
++ name: Log Environment Information
++ command: |
++ echo '*** id ***'
++ id
++ echo '*** cat /proc/cpuinfo ***'
++ cat /proc/cpuinfo
++ echo '*** free -m ***'
++ free -m
++ echo '*** df -m ***'
++ df -m
++ echo '*** ifconfig -a ***'
++ ifconfig -a
++ echo '*** uname -a ***'
++ uname -a
++ echo '*** mount ***'
++ mount
++ echo '*** env ***'
++ env
++ echo '*** java ***'
++ which java
++ java -version
++ - run:
++ name: Run Unit Tests (testclasslist)
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
- ant test-jvm-dtest-forking
++ ant testclasslist -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_build:
+ docker:
+ - image: spod/cassandra-testing-ubuntu1810-java11:20181210
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Clone Cassandra Repository (via git)
+ command: |
+ git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH git://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra
+ - run:
+ name: Build Cassandra
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ cd ~/cassandra
+ # Loop to prevent failure due to maven-ant-tasks not downloading a jar..
+ for x in $(seq 1 3); do
+ ${ANT_HOME}/bin/ant clean realclean jar
+ RETURN="$?"
+ if [ "${RETURN}" -eq "0" ]; then
+ break
+ fi
+ done
+ # Exit, if we didn't build successfully
+ if [ "${RETURN}" -ne "0" ]; then
+ echo "Build failed with exit code: ${RETURN}"
+ exit ${RETURN}
+ fi
+ no_output_timeout: 15m
+ - run:
+ name: Run eclipse-warnings
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ cd ~/cassandra
+ ant eclipse-warnings
+ - persist_to_workspace:
+ root: /home/cassandra
+ paths:
+ - cassandra
+ - .m2
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
j8_dtests-with-vnodes:
docker:
- image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
@@@ -1050,11 -732,11 +1182,11 @@@
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: git://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: master
- - CCM_MAX_HEAP_SIZE: 2048M
- - CCM_HEAP_NEWSIZE: 512M
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- dtest_jars_build:
+ j8_dtest_jars_build:
docker:
- image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
resource_class: medium
@@@ -1161,19 -837,11 +1293,18 @@@ workflows
- utests_stress:
requires:
- start_utests_stress
+ - start_utests_fqltool:
+ type: approval
+ requires:
+ - j8_build
+ - utests_fqltool:
+ requires:
+ - start_utests_fqltool
- start_jvm_upgrade_dtest:
type: approval
+ - j8_dtest_jars_build:
requires:
- - build
+ - j8_build
- - dtest_jars_build:
- requires:
- start_jvm_upgrade_dtest
- j8_jvm_upgrade_dtests:
requires:
diff --cc .circleci/config.yml.LOWRES
index f0f8e03,4108359..6e07fd2
--- a/.circleci/config.yml.LOWRES
+++ b/.circleci/config.yml.LOWRES
@@@ -51,29 -93,8 +92,29 @@@ jobs
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
- parallelism: 1
+ parallelism: 4
steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
- name: Determine Unit Tests to Run
++ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
- set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | cut -c 37-1000000 | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
++ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
- run:
name: Log Environment Information
command: |
@@@ -97,102 -118,41 +138,105 @@@
which java
java -version
- run:
- name: Clone Cassandra Repository (via git)
- command: |
- git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH git://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra
- - run:
- name: Build Cassandra
+ name: Run Unit Tests (testclasslist)
command: |
export PATH=$JAVA_HOME/bin:$PATH
- cd ~/cassandra
- # Loop to prevent failure due to maven-ant-tasks not downloading a jar..
- for x in $(seq 1 3); do
- ${ANT_HOME}/bin/ant clean jar
- RETURN="$?"
- if [ "${RETURN}" -eq "0" ]; then
- break
- fi
- done
- # Exit, if we didn't build successfully
- if [ "${RETURN}" -ne "0" ]; then
- echo "Build failed with exit code: ${RETURN}"
- exit ${RETURN}
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
++ if [ -d ~/dtest_jars ]; then
++ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ ant testclasslist -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_dtests-with-vnodes:
+ docker:
+ - image: spod/cassandra-testing-ubuntu1810-java11:20181210
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
- run:
- name: Run eclipse-warnings
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
- cd ~/cassandra
- ant eclipse-warnings
- - persist_to_workspace:
- root: /home/cassandra
- paths:
- - cassandra
- - .m2
+ pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-re [...]
+ - run:
+ name: Run dtests (j11_with_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the ex [...]
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_with_vnodes_logs
environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
@@@ -492,133 -458,6 +539,174 @@@
- CCM_HEAP_NEWSIZE: 256M
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j11_jvm_dtests:
+ docker:
+ - image: spod/cassandra-testing-ubuntu1810-java11:20181210
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
- name: Run Unit Tests (test-jvm-dtest-forking)
++ name: Determine distributed Tests to Run
++ command: |
++ # reminder: this code (along with all the steps) is independently executed on every circle container
++ # so the goal here is to get the circleci script to return the tests *this* container will run
++ # which we do via the `circleci` cli tool.
++
++ rm -fr ~/cassandra-dtest/upgrade_tests
++ echo "***java tests***"
++
++ # get all of our unit test filenames
++ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt
++
++ # split up the unit tests into groups based on the number of containers we have
++ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
++ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep -v upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
++ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
++ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
++ no_output_timeout: 15m
++ - run:
++ name: Log Environment Information
++ command: |
++ echo '*** id ***'
++ id
++ echo '*** cat /proc/cpuinfo ***'
++ cat /proc/cpuinfo
++ echo '*** free -m ***'
++ free -m
++ echo '*** df -m ***'
++ df -m
++ echo '*** ifconfig -a ***'
++ ifconfig -a
++ echo '*** uname -a ***'
++ uname -a
++ echo '*** mount ***'
++ mount
++ echo '*** env ***'
++ env
++ echo '*** java ***'
++ which java
++ java -version
++ - run:
++ name: Run Unit Tests (testclasslist)
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
- ant test-jvm-dtest-forking
++ ant testclasslist -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_build:
+ docker:
+ - image: spod/cassandra-testing-ubuntu1810-java11:20181210
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Clone Cassandra Repository (via git)
+ command: |
+ git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH git://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra
+ - run:
+ name: Build Cassandra
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ cd ~/cassandra
+ # Loop to prevent failure due to maven-ant-tasks not downloading a jar..
+ for x in $(seq 1 3); do
+ ${ANT_HOME}/bin/ant clean realclean jar
+ RETURN="$?"
+ if [ "${RETURN}" -eq "0" ]; then
+ break
+ fi
+ done
+ # Exit, if we didn't build successfully
+ if [ "${RETURN}" -ne "0" ]; then
+ echo "Build failed with exit code: ${RETURN}"
+ exit ${RETURN}
+ fi
+ no_output_timeout: 15m
+ - run:
+ name: Run eclipse-warnings
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ cd ~/cassandra
+ ant eclipse-warnings
+ - persist_to_workspace:
+ root: /home/cassandra
+ paths:
+ - cassandra
+ - .m2
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: master
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
j8_dtests-with-vnodes:
docker:
- image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
@@@ -1161,19 -837,11 +1293,18 @@@ workflows
- utests_stress:
requires:
- start_utests_stress
+ - start_utests_fqltool:
+ type: approval
+ requires:
+ - j8_build
+ - utests_fqltool:
+ requires:
+ - start_utests_fqltool
- start_jvm_upgrade_dtest:
type: approval
+ - j8_dtest_jars_build:
requires:
- - build
+ - j8_build
- - dtest_jars_build:
- requires:
- start_jvm_upgrade_dtest
- j8_jvm_upgrade_dtests:
requires:
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org