You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2020/07/22 19:06:34 UTC
[airflow] 06/11: Reorganizing of CI tests (#9654)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v1-10-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit d33f6d0bbbfbc4b383b5eaea0773a0c21d7e4057
Author: Jarek Potiuk <ja...@polidea.com>
AuthorDate: Fri Jul 17 10:30:56 2020 +0200
Reorganizing of CI tests (#9654)
* we come back to idea of having one CI workflow
* cancel and openapi are incorporated into that CI workflow
* cancel retrieves workflow id automatically (works for forks)
* static checks are now merged into one job
* less dependencies between jobs so that waiting is minimised
* better name for check if tests should be run
* separated out script for tests should be run check
(cherry picked from commit 496ed6f1b279dc5ca560dec0044c8373531565f4)
---
.github/workflows/ci.yml | 102 ++++++++++++---------
.../get_workflow_id.sh} | 18 ++--
scripts/ci/{ => kubernetes}/ci_run_helm_testing.sh | 2 +-
scripts/ci/libraries/_initialization.sh | 2 +-
...files.sh => ci_check_if_tests_should_be_run.sh} | 69 +++++++-------
scripts/ci/tools/ci_count_changed_files.sh | 19 +++-
6 files changed, 119 insertions(+), 93 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index a849d58..604fa0d 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -40,10 +40,30 @@ env:
jobs:
+ cancel-previous-workflow-run:
+ timeout-minutes: 60
+ name: "Cancel previous workflow run"
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@master
+ - name: Get ci workflow id
+ run: "scripts/ci/cancel/get_workflow_id.sh"
+ env:
+ WORKFLOW: ci
+ GITHUB_TOKEN: ${{ github.token }}
+ GITHUB_REPOSITORY: ${{ github.repositoru }}
+ - name: Cancel workflow ${{ github.workflow }}
+ uses: styfle/cancel-workflow-action@0.3.2
+ with:
+ workflow_id: ${{ env.WORKFLOW_ID }}
+ access_token: ${{ github.token }}
+
static-checks:
timeout-minutes: 60
name: "Checks"
runs-on: ubuntu-latest
+ needs:
+ - cancel-previous-workflow-run
env:
MOUNT_SOURCE_DIR_FOR_STATIC_CHECKS: "true"
CI_JOB_TYPE: "Static checks"
@@ -69,11 +89,12 @@ jobs:
python -m pip install pre-commit \
--constraint requirements/requirements-python${PYTHON_MAJOR_MINOR_VERSION}.txt
./scripts/ci/static_checks/ci_run_static_checks.sh
-
docs:
timeout-minutes: 60
- name: Build docs
+ name: "Build docs"
runs-on: ubuntu-latest
+ needs:
+ - cancel-previous-workflow-run
env:
CI_JOB_TYPE: "Documentation"
steps:
@@ -86,42 +107,25 @@ jobs:
- name: "Build docs"
run: ./scripts/ci/docs/ci_docs.sh
- build-prod-image:
- timeout-minutes: 60
- name: "Build prod image Py${{ matrix.python-version }}"
- runs-on: ubuntu-latest
- strategy:
- matrix:
- python-version: [2.7, 3.5, 3.6, 3.7, 3.8]
- env:
- PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
- CI_JOB_TYPE: "Prod image"
- steps:
- - uses: actions/checkout@master
- - name: "Build PROD image ${{ matrix.python-version }}"
- run: ./scripts/ci/images/ci_prepare_prod_image_on_ci.sh
-
trigger-tests:
- timeout-minutes: 10
- name: "Count changed important files"
+ timeout-minutes: 5
+ name: "Checks if tests should be run"
runs-on: ubuntu-latest
+ needs:
+ - cancel-previous-workflow-run
outputs:
- count: ${{ steps.trigger-tests.outputs.count }}
+ run-tests: ${{ steps.trigger-tests.outputs.run-tests }}
steps:
- uses: actions/checkout@master
- - name: "Get count of changed python files"
- run: |
- set +e
- ./scripts/ci/tools/ci_count_changed_files.sh ${GITHUB_SHA} \
- '^airflow|.github/workflows/|^Dockerfile|^scripts|^chart|^setup.py|^requirements|^tests|^kubernetes_tests'
- echo "::set-output name=count::$?"
+ - name: "Check if tests should be run"
+ run: "./scripts/ci/tools/ci_check_if_tests_should_be_run.sh"
id: trigger-tests
tests-kubernetes:
timeout-minutes: 80
name: "K8s: ${{matrix.kube-mode}} ${{matrix.python-version}} ${{matrix.kubernetes-version}}"
runs-on: ubuntu-latest
- needs: [static-checks, trigger-tests]
+ needs: [trigger-tests]
strategy:
matrix:
python-version: [3.6, 3.7]
@@ -147,8 +151,7 @@ jobs:
KUBERNETES_VERSION: "${{ matrix.kubernetes-version }}"
KIND_VERSION: "${{ matrix.kind-version }}"
HELM_VERSION: "${{ matrix.helm-version }}"
- # For pull requests only run tests when important files changed
- if: needs.trigger-tests.outputs.count != '0' || github.event_name != 'pull_request'
+ if: needs.trigger-tests.outputs.run-tests == 'true' || github.event_name != 'pull_request'
steps:
- uses: actions/checkout@master
- uses: actions/setup-python@v1
@@ -186,7 +189,7 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt')
timeout-minutes: 80
name: "${{matrix.test-type}}:Pg${{matrix.postgres-version}},Py${{matrix.python-version}}"
runs-on: ubuntu-latest
- needs: [static-checks, trigger-tests]
+ needs: [trigger-tests]
strategy:
matrix:
python-version: [2.7, 3.5, 3.6, 3.7, 3.8]
@@ -200,8 +203,7 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt')
RUN_TESTS: "true"
CI_JOB_TYPE: "Tests"
TEST_TYPE: ${{ matrix.test-type }}
- # For pull requests only run tests when important files changed
- if: needs.trigger-tests.outputs.count != '0' || github.event_name != 'pull_request'
+ if: needs.trigger-tests.outputs.run-tests == 'true' || github.event_name != 'pull_request'
steps:
- uses: actions/checkout@master
- uses: actions/setup-python@v1
@@ -218,7 +220,7 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt')
timeout-minutes: 80
name: "${{matrix.test-type}}:MySQL${{matrix.mysql-version}}, Py${{matrix.python-version}}"
runs-on: ubuntu-latest
- needs: [static-checks, trigger-tests]
+ needs: [trigger-tests]
strategy:
matrix:
python-version: [2.7, 3.5, 3.6, 3.7, 3.8]
@@ -232,8 +234,7 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt')
RUN_TESTS: "true"
CI_JOB_TYPE: "Tests"
TEST_TYPE: ${{ matrix.test-type }}
- # For pull requests only run tests when important files changed
- if: needs.trigger-tests.outputs.count != '0' || github.event_name != 'pull_request'
+ if: needs.trigger-tests.outputs.run-tests == 'true' || github.event_name != 'pull_request'
steps:
- uses: actions/checkout@master
- uses: actions/setup-python@v1
@@ -250,7 +251,7 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt')
timeout-minutes: 80
name: "${{matrix.test-type}}:Sqlite Py${{matrix.python-version}}"
runs-on: ubuntu-latest
- needs: [static-checks, trigger-tests]
+ needs: [trigger-tests]
strategy:
matrix:
python-version: [2.7, 3.5, 3.6, 3.7, 3.8]
@@ -262,8 +263,7 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt')
TEST_TYPE: ${{ matrix.test-type }}
RUN_TESTS: "true"
CI_JOB_TYPE: "Tests"
- # For pull requests only run tests when python files changed
- if: needs.trigger-tests.outputs.count != '0' || github.event_name != 'pull_request'
+ if: needs.trigger-tests.outputs.run-tests == 'true' || github.event_name != 'pull_request'
steps:
- uses: actions/checkout@master
- uses: actions/setup-python@v1
@@ -281,7 +281,7 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt')
name: "${{matrix.test-type}}:Pg${{matrix.postgres-version}},Py${{matrix.python-version}}"
runs-on: ubuntu-latest
continue-on-error: true
- needs: [static-checks, trigger-tests]
+ needs: [trigger-tests]
strategy:
matrix:
python-version: [3.6]
@@ -295,8 +295,7 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt')
RUN_TESTS: "true"
CI_JOB_TYPE: "Tests"
TEST_TYPE: ${{ matrix.test-type }}
- # For pull requests only run tests when important files changed
- if: needs.trigger-tests.outputs.count != '0' || github.event_name != 'pull_request'
+ if: needs.trigger-tests.outputs.run-tests == 'true' || github.event_name != 'pull_request'
steps:
- uses: actions/checkout@master
- uses: actions/setup-python@v1
@@ -313,12 +312,14 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt')
timeout-minutes: 5
name: "Checks: Helm tests"
runs-on: ubuntu-latest
+ needs:
+ - cancel-previous-workflow-run
env:
CI_JOB_TYPE: "Tests"
steps:
- uses: actions/checkout@master
- name: "Helm Tests"
- run: ./scripts/ci/ci_run_helm_testing.sh
+ run: ./scripts/ci/kubernetes/ci_run_helm_testing.sh
requirements:
timeout-minutes: 80
@@ -328,6 +329,8 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt')
matrix:
python-version: [2.7, 3.5, 3.6, 3.7, 3.8]
fail-fast: false
+ needs:
+ - cancel-previous-workflow-run
env:
PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
CHECK_REQUIREMENTS_ONLY: true
@@ -343,6 +346,21 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt')
- name: "Generate requirements"
run: ./scripts/ci/requirements/ci_generate_requirements.sh
+ build-prod-image:
+ timeout-minutes: 60
+ name: "Build prod image Py${{ matrix.python-version }}"
+ runs-on: ubuntu-latest
+ strategy:
+ matrix:
+ python-version: [2.7, 3.5, 3.6, 3.7, 3.8]
+ env:
+ PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
+ CI_JOB_TYPE: "Prod image"
+ steps:
+ - uses: actions/checkout@master
+ - name: "Build PROD image ${{ matrix.python-version }}"
+ run: ./scripts/ci/images/ci_prepare_prod_image_on_ci.sh
+
push-prod-images-to-github-cache:
timeout-minutes: 80
name: "Push PROD images"
diff --git a/scripts/ci/ci_run_helm_testing.sh b/scripts/ci/cancel/get_workflow_id.sh
similarity index 67%
copy from scripts/ci/ci_run_helm_testing.sh
copy to scripts/ci/cancel/get_workflow_id.sh
index 0a267d4..4fa6187 100755
--- a/scripts/ci/ci_run_helm_testing.sh
+++ b/scripts/ci/cancel/get_workflow_id.sh
@@ -15,14 +15,10 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
-echo "Running helm tests"
-
-CHART_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/../../chart/"
-
-echo "Chart directory is $CHART_DIR"
-
-docker run -w /airflow-chart -v "$CHART_DIR":/airflow-chart \
- --entrypoint /bin/sh \
- aneeshkj/helm-unittest \
- -c "helm repo add stable https://kubernetes-charts.storage.googleapis.com; helm dependency update ; helm unittest ."
+set -euo pipefail
+echo "Getting workflow id for ${WORKFLOW}. Github Repo: ${GITHUB_REPOSITORY}"
+URL="https://api.github.com/repos/${GITHUB_REPOSITORY}/actions/workflows/${WORKFLOW}.yml"
+echo "Calling URL: ${URL}"
+WORKFLOW_ID=$(curl "Authorization: token ${GITHUB_TOKEN}" "${URL}" | jq '.id')
+echo "Workflow id for ${WORKFLOW}: ${WORKFLOW_ID}"
+echo "::set-env name=WORKFLOW_ID::${WORKFLOW_ID}"
diff --git a/scripts/ci/ci_run_helm_testing.sh b/scripts/ci/kubernetes/ci_run_helm_testing.sh
similarity index 98%
rename from scripts/ci/ci_run_helm_testing.sh
rename to scripts/ci/kubernetes/ci_run_helm_testing.sh
index 0a267d4..d48e298 100755
--- a/scripts/ci/ci_run_helm_testing.sh
+++ b/scripts/ci/kubernetes/ci_run_helm_testing.sh
@@ -18,7 +18,7 @@
echo "Running helm tests"
-CHART_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/../../chart/"
+CHART_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/../../../chart/"
echo "Chart directory is $CHART_DIR"
diff --git a/scripts/ci/libraries/_initialization.sh b/scripts/ci/libraries/_initialization.sh
index d0b14a7..e759b03 100644
--- a/scripts/ci/libraries/_initialization.sh
+++ b/scripts/ci/libraries/_initialization.sh
@@ -280,7 +280,7 @@ function get_environment_for_builds_on_ci() {
fi
elif [[ ${GITHUB_ACTIONS:=} == "true" ]]; then
export CI_TARGET_REPO="${GITHUB_REPOSITORY}"
- export CI_TARGET_BRANCH="${GITHUB_BASE_REF}"
+ export CI_TARGET_BRANCH="${GITHUB_BASE_REF:=${CI_TARGET_BRANCH}}"
export CI_BUILD_ID="${GITHUB_RUN_ID}"
export CI_JOB_ID="${GITHUB_JOB}"
if [[ ${GITHUB_EVENT_NAME:=} == "pull_request" ]]; then
diff --git a/scripts/ci/tools/ci_count_changed_files.sh b/scripts/ci/tools/ci_check_if_tests_should_be_run.sh
similarity index 53%
copy from scripts/ci/tools/ci_count_changed_files.sh
copy to scripts/ci/tools/ci_check_if_tests_should_be_run.sh
index e3d2c9e..5a51048 100755
--- a/scripts/ci/tools/ci_count_changed_files.sh
+++ b/scripts/ci/tools/ci_check_if_tests_should_be_run.sh
@@ -16,40 +16,41 @@
# specific language governing permissions and limitations
# under the License.
-# Returns number of files matching the pattern changed in revision specified
-# Versus the tip of the target branch
-# Parameters
-# $1: Revision to compare
-# $2: Pattern to match
-
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
-get_environment_for_builds_on_ci
-
-git remote add target "https://github.com/${CI_TARGET_REPO}"
-
-git fetch target "${CI_TARGET_BRANCH}:${CI_TARGET_BRANCH}" --depth=1
-
-CHANGED_FILES=$(git diff-tree --no-commit-id --name-only -r "${1}" "${CI_TARGET_BRANCH}" || true)
-
-echo
-echo "Changed files:"
-echo
-echo "${CHANGED_FILES}"
-echo
-
-echo
-echo "Changed files matching the ${2} pattern"
-echo
-echo "${CHANGED_FILES}" | grep -E "${2}" || true
-echo
-
-echo
-echo "Count changed files matching the ${2} pattern"
-echo
-COUNT_CHANGED_FILES=$(echo "${CHANGED_FILES}" | grep -c -E "${2}" || true)
-echo "${COUNT_CHANGED_FILES}"
-echo
-
-exit "${COUNT_CHANGED_FILES}"
+CHANGED_FILES_PATTERNS=(
+ "^airflow"
+ "^.github/workflows/"
+ "^Dockerfile"
+ "^scripts"
+ "^chart"
+ "^setup.py"
+ "^requirements"
+ "^tests"
+ "^kubernetes_tests"
+)
+
+CHANGED_FILES_REGEXP=""
+
+SEPARATOR=""
+for PATTERN in "${CHANGED_FILES_PATTERNS[@]}"
+do
+ CHANGED_FILES_REGEXP="${CHANGED_FILES_REGEXP}${SEPARATOR}${PATTERN}"
+ SEPARATOR="|"
+done
+
+echo
+echo "GitHub SHA: ${GITHUB_SHA}"
+echo
+
+set +e
+"${SCRIPTS_CI_DIR}/tools/ci_count_changed_files.sh" "${CHANGED_FILES_REGEXP}"
+COUNT_CHANGED_FILES=$?
+set -e
+
+if [[ ${COUNT_CHANGED_FILES} == "0" ]]; then
+ echo "::set-output name=run-tests::false"
+else
+ echo "::set-output name=run-tests::true"
+fi
diff --git a/scripts/ci/tools/ci_count_changed_files.sh b/scripts/ci/tools/ci_count_changed_files.sh
index e3d2c9e..ef16eb1 100755
--- a/scripts/ci/tools/ci_count_changed_files.sh
+++ b/scripts/ci/tools/ci_count_changed_files.sh
@@ -27,11 +27,22 @@
get_environment_for_builds_on_ci
+if [[ ${CI_EVENT_TYPE} == "push" ]]; then
+ echo
+ echo "Always run all tests on push"
+ echo
+ exit 1
+fi
+
git remote add target "https://github.com/${CI_TARGET_REPO}"
git fetch target "${CI_TARGET_BRANCH}:${CI_TARGET_BRANCH}" --depth=1
-CHANGED_FILES=$(git diff-tree --no-commit-id --name-only -r "${1}" "${CI_TARGET_BRANCH}" || true)
+echo
+echo "Retrieve changed files from ${GITHUB_SHA} comparing to ${CI_TARGET_BRANCH}"
+echo
+
+CHANGED_FILES=$(git diff-tree --no-commit-id --name-only -r "${GITHUB_SHA}" "${CI_TARGET_BRANCH}" || true)
echo
echo "Changed files:"
@@ -40,13 +51,13 @@ echo "${CHANGED_FILES}"
echo
echo
-echo "Changed files matching the ${2} pattern"
+echo "Changed files matching the ${1} pattern"
echo
-echo "${CHANGED_FILES}" | grep -E "${2}" || true
+echo "${CHANGED_FILES}" | grep -E "${1}" || true
echo
echo
-echo "Count changed files matching the ${2} pattern"
+echo "Count changed files matching the ${1} pattern"
echo
COUNT_CHANGED_FILES=$(echo "${CHANGED_FILES}" | grep -c -E "${2}" || true)
echo "${COUNT_CHANGED_FILES}"