You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by ab...@apache.org on 2023/01/13 09:17:09 UTC
[druid] branch master updated: Migrate jdk8 unit tests from Travis to GHA (#13518)
This is an automated email from the ASF dual-hosted git repository.
abhishek pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new 4368b3a071 Migrate jdk8 unit tests from Travis to GHA (#13518)
4368b3a071 is described below
commit 4368b3a071719b50e8341626d967b8ae77caa786
Author: Tejaswini Bandlamudi <96...@users.noreply.github.com>
AuthorDate: Fri Jan 13 14:46:58 2023 +0530
Migrate jdk8 unit tests from Travis to GHA (#13518)
* migrate UTs form Travis to GHA
* update permissions
* rename file
* set fetch depth to 1
* debugs remote branches
* test with github.ref variable
* fetch github.base_ref for diff
* nit
* test git diff
* run tests
* test code coverage failure scenario
* nit
* nit
* revert code changes
* revert code changes
* Setup diff-test-coverage before tests
* build distribution module at end in packaging check
* nit
* remove redundant steps in static-checks workflow
* drop jdk8 unit tests from Travis
---
.github/scripts/unit_tests_script.sh | 77 ++++++++++++++++++++++++++++
.github/workflows/static-checks.yml | 10 ++--
.github/workflows/unit-tests.yml | 98 ++++++++++++++++++++++++++++++++++++
.travis.yml | 57 ++++++---------------
4 files changed, 194 insertions(+), 48 deletions(-)
diff --git a/.github/scripts/unit_tests_script.sh b/.github/scripts/unit_tests_script.sh
new file mode 100755
index 0000000000..f27bc4bad9
--- /dev/null
+++ b/.github/scripts/unit_tests_script.sh
@@ -0,0 +1,77 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+unset _JAVA_OPTIONS
+
+# Set MAVEN_OPTS for Surefire launcher.
+MAVEN_OPTS='-Xmx2048m' ${MVN} test -pl ${MAVEN_PROJECTS} \
+${MAVEN_SKIP} -Ddruid.generic.useDefaultValueForNull=${DRUID_USE_DEFAULT_VALUE_FOR_NULL}
+sh -c "dmesg | egrep -i '(oom|out of memory|kill process|killed).*' -C 1 || exit 0"
+free -m
+${MVN} -pl ${MAVEN_PROJECTS} jacoco:report
+
+# Determine the modified files that match the maven projects being tested. We use maven project lists that
+# either exclude (starts with "!") or include (does not start with "!"), so both cases need to be handled.
+# If the build is triggered by a tag, an error will be printed, but `all_files` will be correctly set to empty
+# so that the coverage check is skipped.
+all_files="$(git diff --name-only origin/${GITHUB_BASE_REF}...HEAD | grep "\.java$" || [[ $? == 1 ]])"
+
+echo "Changed files:"
+for f in ${all_files}
+do
+ echo $f # for debugging
+done
+
+if [[ "${MAVEN_PROJECTS}" = \!* ]]
+then
+ regex="${MAVEN_PROJECTS:1}"
+ regex="^${regex//,\!/\\|^}"
+ project_files="$(echo "${all_files}" | grep -v "${regex}" || [[ $? == 1 ]])"
+else
+ regex="^${MAVEN_PROJECTS//,/\\|^}"
+ project_files="$(echo "${all_files}" | grep "${regex}" || [[ $? == 1 ]])"
+fi
+
+echo "Changed files in current project:"
+for f in ${project_files}
+do
+ echo $f # for debugging
+done
+
+# Check diff code coverage for the maven projects being tested (retry install in case of network error).
+# Currently, the function coverage check is not reliable, so it is disabled.
+if [ -n "${project_files}" ]
+then
+ git diff origin/${GITHUB_BASE_REF}...HEAD -- ${project_files} |
+ node_modules/.bin/diff-test-coverage \
+ --coverage "**/target/site/jacoco/jacoco.xml" \
+ --type jacoco \
+ --line-coverage 50 \
+ --branch-coverage 50 \
+ --function-coverage 0 \
+ --log-template "coverage-lines-complete" \
+ --log-template "coverage-files-complete" \
+ --log-template "totals-complete" \
+ --log-template "errors" \
+ -- ||
+ { printf "\n\n****FAILED****\nDiff code coverage check failed. To view coverage report, run 'mvn clean test jacoco:report' and open 'target/site/jacoco/index.html'\nFor more details on how to run code coverage locally, follow instructions here - https://github.com/apache/druid/blob/master/dev/code-review/code-coverage.md#running-code-coverage-locally\n\n" && false; }
+fi
+
+{ for i in 1 2 3; do curl -o codecov.sh -s https://codecov.io/bash && break || sleep 15; done }
+{ for i in 1 2 3; do bash codecov.sh -X gcov && break || sleep 15; done }
diff --git a/.github/workflows/static-checks.yml b/.github/workflows/static-checks.yml
index 37e7a22258..f247bf01a1 100644
--- a/.github/workflows/static-checks.yml
+++ b/.github/workflows/static-checks.yml
@@ -59,8 +59,10 @@ jobs:
- name: packaging check
run: |
./.github/scripts/setup_generate_license.sh
- ${MVN} clean install -Prat -Pdist -Pbundle-contrib-exts --fail-at-end \
- -pl '!benchmarks' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} -Dweb.console.skip=false -T1C
+ ${MVN} clean install -Prat -Pbundle-contrib-exts --fail-at-end \
+ -pl '!benchmarks, !distribution' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} -Dweb.console.skip=false -T1C
+ ${MVN} install -Prat -Pdist -Pbundle-contrib-exts --fail-at-end \
+ -pl 'distribution' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} -Dweb.console.skip=false -T1C
- name: script checks
# who watches the watchers?
@@ -163,10 +165,6 @@ jobs:
if: ${{ matrix.java == 'jdk8' }}
run: |
./.github/scripts/setup_generate_license.sh
- sudo apt-get update && sudo apt-get install python3 -y
- curl https://bootstrap.pypa.io/pip/3.5/get-pip.py | sudo -H python3
- pip3 install wheel # install wheel first explicitly
- pip3 install --upgrade pyyaml
web-console/script/druid build
web-console/script/druid start
(cd web-console && npm run test-e2e)
diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml
new file mode 100644
index 0000000000..9635fb44b7
--- /dev/null
+++ b/.github/workflows/unit-tests.yml
@@ -0,0 +1,98 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: (openjdk8) Unit Tests CI
+on:
+ push:
+ branches:
+ - master
+ - /^\d+\.\d+\.\d+(-\S*)?$/ # release branches
+ pull_request:
+ branches:
+ - master
+ - /^\d+\.\d+\.\d+(-\S*)?$/ # release branches
+
+concurrency:
+ group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}'
+ cancel-in-progress: true
+
+env:
+ MVN: mvn -B
+ MAVEN_SKIP: -P skip-static-checks -Dweb.console.skip=true -Dmaven.javadoc.skip=true
+ MAVEN_SKIP_TESTS: -P skip-tests
+ MAVEN_OPTS: -Xmx3000m
+ FORCE_COLOR: 2
+
+jobs:
+ unit-tests:
+ strategy:
+ matrix:
+ DRUID_USE_DEFAULT_VALUE_FOR_NULL: [ false, true ]
+ runs-on: ubuntu-latest
+ steps:
+ - name: checkout branch
+ uses: actions/checkout@v3
+ with:
+ fetch-depth: 0
+
+ - name: setup jdk8
+ uses: actions/setup-java@v3
+ with:
+ distribution: 'zulu'
+ java-version: 8
+ cache: 'maven'
+
+ - name: maven install
+ run: |
+ echo 'Running Maven install...' &&
+ ${MVN} clean install -q -ff -pl '!distribution,!:druid-it-image,!:druid-it-cases' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} -T1C &&
+ ${MVN} install -q -ff -pl 'distribution' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS}
+
+ - name: setup variables
+ run: |
+ export base_ref=${{ github.base_ref }}
+ echo "GITHUB_BASE_REF=${base_ref}" >> $GITHUB_ENV
+ export druid_use_default_value_for_null=${{ matrix.DRUID_USE_DEFAULT_VALUE_FOR_NULL }}
+ echo "DRUID_USE_DEFAULT_VALUE_FOR_NULL=${druid_use_default_value_for_null}" >> $GITHUB_ENV
+
+ - name: fetch base branch for test coverage
+ if: ${{ github.base_ref != '' }}
+ run: |
+ # Add merge target branch to determine diff.
+ # This is not needed for build triggered by tags, since there will be no code diff.
+ git remote set-branches --add origin ${GITHUB_BASE_REF} && git fetch
+
+ - name: Setup diff-test-coverage
+ run: npm install @connectis/diff-test-coverage@1.5.3
+
+ - name: "(openjdk8) indexing modules test"
+ env:
+ MAVEN_PROJECTS: indexing-hadoop,indexing-service,extensions-core/kafka-indexing-service,extensions-core/kinesis-indexing-service
+ run: ./.github/scripts/unit_tests_script.sh
+
+ - name: "(openjdk8) processing module test"
+ env:
+ MAVEN_PROJECTS: processing
+ run: ./.github/scripts/unit_tests_script.sh
+
+ - name: "(openjdk8) server module test"
+ env:
+ MAVEN_PROJECTS: server
+ run: ./.github/scripts/unit_tests_script.sh
+
+ - name: "(openjdk8) other modules test"
+ env:
+ MAVEN_PROJECTS: '!processing,!indexing-hadoop,!indexing-service,!extensions-core/kafka-indexing-service,!extensions-core/kinesis-indexing-service,!server,!web-console,!integration-tests,!:druid-it-tools,!:druid-it-image,!:druid-it-cases'
+ run: ./.github/scripts/unit_tests_script.sh
diff --git a/.travis.yml b/.travis.yml
index 003331be2e..c9afd77e14 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -49,7 +49,7 @@ addons:
install: ./check_test_suite.py && travis_terminate 0 || echo 'Running Maven install...' && MAVEN_OPTS='-Xmx3000m' travis_wait 15 ${MVN} clean install -q -ff -pl '!distribution,!:druid-it-image,!:druid-it-cases' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} -T1C && ${MVN} install -q -ff -pl 'distribution' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS}
# There are 3 stages of tests
-# 1. Tests - phase 1
+# 1. Tests - phase 1 (completely migrated to github workflows)
# 2. Tests - phase 2
# 3. cron
#
@@ -71,8 +71,9 @@ stages:
jobs:
include:
- &test_processing_module
- name: "(openjdk8) processing module test"
- stage: Tests - phase 1
+ name: "(openjdk11) processing module test"
+ stage: Tests - phase 2
+ jdk: openjdk11
env:
- MAVEN_PROJECTS='processing'
before_script:
@@ -131,11 +132,6 @@ jobs:
- travis_retry curl -o codecov.sh -s https://codecov.io/bash
- travis_retry bash codecov.sh -X gcov
- - <<: *test_processing_module
- name: "(openjdk11) processing module test"
- stage: Tests - phase 2
- jdk: openjdk11
-
- <<: *test_processing_module
name: "(openjdk17) processing module test"
stage: Tests - phase 2
@@ -143,15 +139,11 @@ jobs:
- &test_processing_module_sqlcompat
<<: *test_processing_module
- name: "(openjdk8) processing module test (SQL Compatibility)"
- stage: Tests - phase 1
- before_script: &setup_sqlcompat
- - export DRUID_USE_DEFAULT_VALUE_FOR_NULL=false
-
- - <<: *test_processing_module_sqlcompat
name: "(openjdk11) processing module test (SQL Compatibility)"
stage: Tests - phase 2
jdk: openjdk11
+ before_script: &setup_sqlcompat
+ - export DRUID_USE_DEFAULT_VALUE_FOR_NULL=false
- <<: *test_processing_module_sqlcompat
name: "(openjdk17) processing module test (SQL Compatibility)"
@@ -160,14 +152,11 @@ jobs:
- &test_indexing_module
<<: *test_processing_module
- name: "(openjdk8) indexing modules test"
- env:
- - MAVEN_PROJECTS='indexing-hadoop,indexing-service,extensions-core/kafka-indexing-service,extensions-core/kinesis-indexing-service'
-
- - <<: *test_indexing_module
name: "(openjdk11) indexing modules test"
stage: Tests - phase 2
jdk: openjdk11
+ env:
+ - MAVEN_PROJECTS='indexing-hadoop,indexing-service,extensions-core/kafka-indexing-service,extensions-core/kinesis-indexing-service'
- <<: *test_indexing_module
name: "(openjdk17) indexing modules test"
@@ -176,14 +165,10 @@ jobs:
- &test_indexing_module_sqlcompat
<<: *test_indexing_module
- name: "(openjdk8) indexing modules test (SQL Compatibility)"
- stage: Tests - phase 1
- before_script: *setup_sqlcompat
-
- - <<: *test_indexing_module_sqlcompat
name: "(openjdk11) indexing modules test (SQL Compatibility)"
stage: Tests - phase 2
jdk: openjdk11
+ before_script: *setup_sqlcompat
- <<: *test_indexing_module_sqlcompat
name: "(openjdk17) indexing modules test (SQL Compatibility)"
@@ -192,14 +177,11 @@ jobs:
- &test_server_module
<<: *test_processing_module
- name: "(openjdk8) server module test"
- env:
- - MAVEN_PROJECTS='server'
-
- - <<: *test_server_module
name: "(openjdk11) server module test"
stage: Tests - phase 2
jdk: openjdk11
+ env:
+ - MAVEN_PROJECTS='server'
- <<: *test_server_module
name: "(openjdk17) server module test"
@@ -208,13 +190,10 @@ jobs:
- &test_server_module_sqlcompat
<<: *test_server_module
- name: "(openjdk8) server module test (SQL Compatibility)"
- before_script: *setup_sqlcompat
-
- - <<: *test_server_module_sqlcompat
name: "(openjdk11) server module test (SQL Compatibility)"
stage: Tests - phase 2
jdk: openjdk11
+ before_script: *setup_sqlcompat
- <<: *test_server_module_sqlcompat
name: "(openjdk17) server module test (SQL Compatibility)"
@@ -223,14 +202,11 @@ jobs:
- &test_other_modules
<<: *test_processing_module
- name: "(openjdk8) other modules test"
- env:
- - MAVEN_PROJECTS='!processing,!indexing-hadoop,!indexing-service,!extensions-core/kafka-indexing-service,!extensions-core/kinesis-indexing-service,!server,!web-console,!integration-tests,!:druid-it-tools,!:druid-it-image,!:druid-it-cases'
-
- - <<: *test_other_modules
name: "(openjdk11) other modules test"
stage: Tests - phase 2
jdk: openjdk11
+ env:
+ - MAVEN_PROJECTS='!processing,!indexing-hadoop,!indexing-service,!extensions-core/kafka-indexing-service,!extensions-core/kinesis-indexing-service,!server,!web-console,!integration-tests,!:druid-it-tools,!:druid-it-image,!:druid-it-cases'
- <<: *test_other_modules
name: "(openjdk17) other modules test"
@@ -239,13 +215,10 @@ jobs:
- &test_other_modules_sqlcompat
<<: *test_other_modules
- name: "(openjdk8) other modules test (SQL Compatibility)"
- before_script: *setup_sqlcompat
-
- - <<: *test_other_modules_sqlcompat
name: "(openjdk11) other modules test (SQL Compatibility)"
stage: Tests - phase 2
jdk: openjdk11
+ before_script: *setup_sqlcompat
- <<: *test_other_modules_sqlcompat
name: "(openjdk17) other modules test (SQL Compatibility)"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org