You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2020/10/06 18:10:31 UTC
[airflow] 03/05: Improve running and canceliling of the
PR-triggered builds. (#11268)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v1-10-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit e3d521fbfefba9dc490a29b4f7e40508629524c7
Author: Jarek Potiuk <ja...@polidea.com>
AuthorDate: Sun Oct 4 22:53:18 2020 +0200
Improve running and canceliling of the PR-triggered builds. (#11268)
The PR builds are now better handled with regards to both
running (using merge-request) and canceling (with cancel notifications).
First of all we are using merged commit from the PR, not the original commit
from the PR.
Secondly - the workflow run notifies the original PR with comment
stating that the image is being built in a separate workflow -
including the link to that workflow.
Thirdly - when canceling duplicate PRs or PRs with failed
jobs, the workflow will add a comment to the PR stating the
reason why the PR is being cancelled.
Last but not least, we also add cancel job for the CodeQL duplicate
messages. They run for ~ 12 miinutes so it makes perfect sense to
also cancel those CodeQL jobs for which someone pushed fixups in a
quick succession.
Fixes: #10471
(cherry picked from commit a4478f5665688afb3e112357f55b90f9838f83ab)
---
.github/workflows/build-images-workflow-run.yml | 40 ++++++++++++++++++++-----
.github/workflows/codeql-cancel.yml | 21 +++++++++++++
2 files changed, 53 insertions(+), 8 deletions(-)
diff --git a/.github/workflows/build-images-workflow-run.yml b/.github/workflows/build-images-workflow-run.yml
index 82b1474..9976eac 100644
--- a/.github/workflows/build-images-workflow-run.yml
+++ b/.github/workflows/build-images-workflow-run.yml
@@ -50,6 +50,8 @@ jobs:
sourceHeadRepo: ${{ steps.cancel.outputs.sourceHeadRepo }}
sourceHeadBranch: ${{ steps.cancel.outputs.sourceHeadBranch }}
sourceHeadSha: ${{ steps.cancel.outputs.sourceHeadSha }}
+ mergeCommitSha: ${{ steps.cancel.outputs.mergeCommitSha }}
+ targetCommitSha: ${{ steps.cancel.outputs.targetCommitSha }}
sourceEvent: ${{ steps.cancel.outputs.sourceEvent }}
cacheDirective: ${{ steps.cache-directive.outputs.docker-cache }}
buildImages: ${{ steps.build-images.outputs.buildImages }}
@@ -57,12 +59,18 @@ jobs:
if: github.repository == 'apache/airflow' || github.event.workflow_run.event != 'schedule'
steps:
- name: "Cancel duplicated 'CI Build' runs"
- uses: potiuk/cancel-workflow-runs@v2
+ uses: potiuk/cancel-workflow-runs@v3
id: cancel
with:
token: ${{ secrets.GITHUB_TOKEN }}
cancelMode: duplicates
sourceRunId: ${{ github.event.workflow_run.id }}
+ notifyPRCancel: true
+ notifyPRMessageStart: |
+ The CI and PROD Docker Images for the build are prepared in a separate "Build Image" workflow,
+ that you will not see in the list of checks (you will see "Wait for images" jobs instead).
+
+ You can checks the status of those images in
- name: "Output BUILD_IMAGES"
id: build-images
run: |
@@ -75,10 +83,11 @@ jobs:
# in GitHub Actions, we have to use Job names to match Event/Repo/Branch from the
# build-info step there to find the duplicates ¯\_(ツ)_/¯.
- uses: potiuk/cancel-workflow-runs@v2
+ uses: potiuk/cancel-workflow-runs@v3
with:
cancelMode: namedJobs
token: ${{ secrets.GITHUB_TOKEN }}
+ notifyPRCancel: true
jobNameRegexps: >
[".*Event: ${{ steps.cancel.outputs.sourceEvent }}
Repo: ${{ steps.cancel.outputs.sourceHeadRepo }}
@@ -92,12 +101,13 @@ jobs:
# can cancel all the matching "Build Images" workflow runs in the two following steps.
# Yeah. Adding to the complexity ¯\_(ツ)_/¯.
- uses: potiuk/cancel-workflow-runs@v2
+ uses: potiuk/cancel-workflow-runs@v3
id: cancel-failed
with:
token: ${{ secrets.GITHUB_TOKEN }}
cancelMode: failedJobs
sourceRunId: ${{ github.event.workflow_run.id }}
+ notifyPRCancel: true
jobNameRegexps: >
["^Static checks.*", "^Build docs$", "^Spell check docs$", "^Backport packages$",
"^Checks: Helm tests$", "^Test OpenAPI*"]
@@ -128,10 +138,11 @@ jobs:
# followed by one of the run ids. Yes I know it's super complex ¯\_(ツ)_/¯.
if: env.BUILD_IMAGES == 'true' && steps.cancel-failed.outputs.cancelledRuns != '[]'
- uses: potiuk/cancel-workflow-runs@v2
+ uses: potiuk/cancel-workflow-runs@v3
with:
cancelMode: namedJobs
token: ${{ secrets.GITHUB_TOKEN }}
+ notifyPRCancel: true
jobNameRegexps: ${{ steps.extract-cancelled-failed-runs.outputs.matching-regexp }}
- name: "Set Docker Cache Directive"
id: cache-directive
@@ -165,6 +176,8 @@ jobs:
Source Run id: ${{ github.event.workflow_run.id }}
Sha: ${{ github.sha }}
Source Sha: ${{ needs.cancel-workflow-runs.outputs.sourceHeadSha }}
+ Merge commit Sha: ${{ needs.cancel-workflow-runs.outputs.mergeCommitSha }}
+ Target commit Sha: ${{ needs.cancel-workflow-runs.outputs.targetCommitSha }}
runs-on: ubuntu-latest
needs: [cancel-workflow-runs]
env:
@@ -187,6 +200,8 @@ jobs:
Source Run id: ${{ github.event.workflow_run.id }}
Sha: ${{ github.sha }}
Source Sha: ${{ needs.cancel-workflow-runs.outputs.sourceHeadSha }}
+ Merge commit Sha: ${{ needs.cancel-workflow-runs.outputs.mergeCommitSha }}
+ Target commit Sha: ${{ needs.cancel-workflow-runs.outputs.targetCommitSha }}
run: |
printenv
- name: Set versions
@@ -232,9 +247,11 @@ jobs:
Source Run id: ${{ github.event.workflow_run.id }}
Sha: ${{ github.sha }}
Source Sha: ${{ needs.cancel-workflow-runs.outputs.sourceHeadSha }}
+ Merge commit Sha: ${{ needs.cancel-workflow-runs.outputs.mergeCommitSha }}
+ Target commit Sha: ${{ needs.cancel-workflow-runs.outputs.targetCommitSha }}
uses: actions/checkout@v2
with:
- ref: ${{ needs.cancel-workflow-runs.outputs.sourceHeadSha }}
+ ref: ${{ needs.cancel-workflow-runs.outputs.targetCommitSha }}
- name: "Retrieve DEFAULTS from the _initialization.sh"
# We cannot "source" the script here because that would be a security problem (we cannot run
# any code that comes from the sources coming from the PR. Therefore we extract the
@@ -290,10 +307,17 @@ jobs:
- name: "Push PROD images ${{ matrix.python-version }}:${{ github.event.workflow_run.id }}"
run: ./scripts/ci/images/ci_push_production_images.sh
if: matrix.image-type == 'PROD' && steps.defaults.outputs.proceed == 'true'
- - name: "Canceling the CI Build source workflow in case of failure!"
- if: cancelled() || failure()
- uses: potiuk/cancel-workflow-runs@v2
+
+ cancel-on-build-failure:
+ name: "Cancel 'CI Build' jobs on build image failure"
+ runs-on: ubuntu-latest
+ if: cancelled() || failure()
+ needs: [build-images]
+ steps:
+ - name: "Canceling the 'CI Build' source workflow in case of failure!"
+ uses: potiuk/cancel-workflow-runs@v3
with:
token: ${{ secrets.GITHUB_TOKEN }}
cancelMode: self
+ notifyPRCancel: true
sourceRunId: ${{ github.event.workflow_run.id }}
diff --git a/.github/workflows/codeql-cancel.yml b/.github/workflows/codeql-cancel.yml
new file mode 100644
index 0000000..216fed1
--- /dev/null
+++ b/.github/workflows/codeql-cancel.yml
@@ -0,0 +1,21 @@
+---
+name: "Cancel duplicate CodeQL workflows"
+on:
+ workflow_run:
+ workflows: ["CodeQL"]
+ types: ['requested']
+jobs:
+ cancel-workflow-runs:
+ timeout-minutes: 10
+ name: "Cancel workflow runs"
+ runs-on: ubuntu-latest
+ if: github.repository == 'apache/airflow' || github.event.workflow_run.event != 'schedule'
+ steps:
+ - name: "Cancel duplicated 'CodeQL' runs"
+ uses: potiuk/cancel-workflow-runs@v3
+ id: cancel
+ with:
+ token: ${{ secrets.GITHUB_TOKEN }}
+ cancelMode: duplicates
+ sourceRunId: ${{ github.event.workflow_run.id }}
+ notifyPRCancel: true