You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ka...@apache.org on 2021/01/21 16:01:24 UTC
[airflow] 04/04: Run "third party" github actions from submodules
instead (#13514)
This is an automated email from the ASF dual-hosted git repository.
kaxilnaik pushed a commit to branch v1-10-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit d9c973bda1d14474edb714796366bc0060e4cbbc
Author: Ash Berlin-Taylor <as...@firemirror.com>
AuthorDate: Mon Jan 11 10:38:15 2021 +0000
Run "third party" github actions from submodules instead (#13514)
Rather than having to mirror all the repos we can instead use
git submodules to pull in the third party actions we want to use - with
recent(ish) changes in review for submodules on GitHub we still get the
same "review/audit" visibility for changes, but this way we don't have
to either "pollute" our repo with the actions code, nor do we have to
maintain a fork of the third party action.
(cherry picked from commit f115983550833b1516ce58e66b49fe4fb1a0eff7)
---
.github/actions/cancel-workflow-runs | 1 +
.github/actions/checks-action | 1 +
.github/actions/codecov-action | 1 +
.github/actions/configure-aws-credentials | 1 +
.github/actions/get-workflow-origin | 1 +
.github/actions/github-push-action | 1 +
.github/actions/label-when-approved-action | 1 +
.github/workflows/build-images-workflow-run.yml | 57 ++++++++++++++--------
.github/workflows/ci.yml | 22 ++++++---
.../workflows/label_when_reviewed_workflow_run.yml | 24 ++++-----
.gitmodules | 21 ++++++++
.pre-commit-config.yaml | 2 +-
12 files changed, 94 insertions(+), 39 deletions(-)
diff --git a/.github/actions/cancel-workflow-runs b/.github/actions/cancel-workflow-runs
new file mode 160000
index 0000000..953e057
--- /dev/null
+++ b/.github/actions/cancel-workflow-runs
@@ -0,0 +1 @@
+Subproject commit 953e057dc81d3458935a18d1184c386b0f6b5738
diff --git a/.github/actions/checks-action b/.github/actions/checks-action
new file mode 160000
index 0000000..9f02872
--- /dev/null
+++ b/.github/actions/checks-action
@@ -0,0 +1 @@
+Subproject commit 9f02872da71b6f558c6a6f190f925dde5e4d8798
diff --git a/.github/actions/codecov-action b/.github/actions/codecov-action
new file mode 160000
index 0000000..1fc7722
--- /dev/null
+++ b/.github/actions/codecov-action
@@ -0,0 +1 @@
+Subproject commit 1fc7722ded4708880a5aea49f2bfafb9336f0c8d
diff --git a/.github/actions/configure-aws-credentials b/.github/actions/configure-aws-credentials
new file mode 160000
index 0000000..e97d7fb
--- /dev/null
+++ b/.github/actions/configure-aws-credentials
@@ -0,0 +1 @@
+Subproject commit e97d7fbc8e0e5af69631c13daa0f4b5a8d88165b
diff --git a/.github/actions/get-workflow-origin b/.github/actions/get-workflow-origin
new file mode 160000
index 0000000..588cc14
--- /dev/null
+++ b/.github/actions/get-workflow-origin
@@ -0,0 +1 @@
+Subproject commit 588cc14f9f1cdf1b8be3db816855e96422204fec
diff --git a/.github/actions/github-push-action b/.github/actions/github-push-action
new file mode 160000
index 0000000..40bf560
--- /dev/null
+++ b/.github/actions/github-push-action
@@ -0,0 +1 @@
+Subproject commit 40bf560936a8022e68a3c00e7d2abefaf01305a6
diff --git a/.github/actions/label-when-approved-action b/.github/actions/label-when-approved-action
new file mode 160000
index 0000000..4c5190f
--- /dev/null
+++ b/.github/actions/label-when-approved-action
@@ -0,0 +1 @@
+Subproject commit 4c5190fec5661e98d83f50bbd4ef9ebb48bd1194
diff --git a/.github/workflows/build-images-workflow-run.yml b/.github/workflows/build-images-workflow-run.yml
index bed5dfc..5c85cb4 100644
--- a/.github/workflows/build-images-workflow-run.yml
+++ b/.github/workflows/build-images-workflow-run.yml
@@ -56,14 +56,19 @@ jobs:
cacheDirective: ${{ steps.cache-directive.outputs.docker-cache }}
buildImages: ${{ steps.build-images.outputs.buildImages }}
steps:
+ - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+ uses: actions/checkout@v2
+ with:
+ persist-credentials: false
+ submodules: recursive
- name: "Get information about the original trigger of the run"
- uses: apache/airflow-get-workflow-origin@588cc14f9f1cdf1b8be3db816855e96422204fec # v1_3
+ uses: ./.github/actions/get-workflow-origin
id: source-run-info
with:
token: ${{ secrets.GITHUB_TOKEN }}
sourceRunId: ${{ github.event.workflow_run.id }}
- name: "Cancel duplicated 'CI Build' runs"
- uses: apache/airflow-cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738 # v4_7
+ uses: ./.github/actions/cancel-workflow-runs
with:
token: ${{ secrets.GITHUB_TOKEN }}
cancelMode: allDuplicates
@@ -80,7 +85,7 @@ jobs:
# trick ¯\_(ツ)_/¯. We name the build-info job appropriately
# and then we try to find and cancel all the jobs with the same Event + Repo + Branch as the
# current Event/Repo/Branch combination.
- uses: apache/airflow-cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738 # v4_7
+ uses: ./.github/actions/cancel-workflow-runs
with:
cancelMode: namedJobs
token: ${{ secrets.GITHUB_TOKEN }}
@@ -96,7 +101,7 @@ jobs:
# We also produce list of canceled "CI Build' runs as output, so that we
# can cancel all the matching "Build Images" workflow runs in the two following steps.
# Yeah. Adding to the complexity ¯\_(ツ)_/¯.
- uses: apache/airflow-cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738 # v4_7
+ uses: ./.github/actions/cancel-workflow-runs
id: cancel-failed
with:
token: ${{ secrets.GITHUB_TOKEN }}
@@ -129,14 +134,14 @@ jobs:
# it to cancel any jobs that have matching names containing Source Run Id:
# followed by one of the run ids. Yes I know it's super complex ¯\_(ツ)_/¯.
if: env.BUILD_IMAGES == 'true' && steps.cancel-failed.outputs.cancelledRuns != '[]'
- uses: apache/airflow-cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738 # v4_7
+ uses: ./.github/actions/cancel-workflow-runs
with:
cancelMode: namedJobs
token: ${{ secrets.GITHUB_TOKEN }}
notifyPRCancel: true
jobNameRegexps: ${{ steps.extract-cancelled-failed-runs.outputs.matching-regexp }}
- name: "Cancel duplicated 'CodeQL' runs"
- uses: apache/airflow-cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738 # v4_7
+ uses: ./.github/actions/cancel-workflow-runs
id: cancel
with:
token: ${{ secrets.GITHUB_TOKEN }}
@@ -156,7 +161,7 @@ jobs:
# trick ¯\_(ツ)_/¯. We name the build-info job appropriately and then we try to match
# all the jobs with the same Event + Repo + Branch match and cancel all the duplicates for those
# This might cancel own run, so this is the last step in the job
- uses: apache/airflow-cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738 # v4_7
+ uses: ./.github/actions/cancel-workflow-runs
with:
cancelMode: allDuplicatedNamedJobs
token: ${{ secrets.GITHUB_TOKEN }}
@@ -307,8 +312,19 @@ jobs:
else
echo "::set-output name=proceed::false"
fi
+ - name: >
+ Checkout "${{ needs.cancel-workflow-runs.outputs.targetBranch }}" branch to 'main-airflow' folder
+ to use ci/scripts from there.
+ uses: actions/checkout@v2
+ with:
+ path: "main-airflow"
+ ref: "${{ needs.cancel-workflow-runs.outputs.targetBranch }}"
+ persist-credentials: false
+ submodules: recursive
+ if: steps.defaults.outputs.proceed == 'true'
- name: Initiate Github Checks for Building image
- uses: apache/airflow-checks-action@9f02872da71b6f558c6a6f190f925dde5e4d8798 # v1.1.0
+ # Use the submodule from main, not the PR branch
+ uses: ./main-airflow/.github/actions/checks-action
id: build-image-check
with:
token: ${{ secrets.GITHUB_TOKEN }}
@@ -322,15 +338,6 @@ jobs:
[Image Build](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})
for details" }
if: steps.defaults.outputs.proceed == 'true'
- - name: >
- Checkout "${{ needs.cancel-workflow-runs.outputs.targetBranch }}" branch to 'main-airflow' folder
- to use ci/scripts from there.
- uses: actions/checkout@v2
- with:
- path: "main-airflow"
- ref: "${{ needs.cancel-workflow-runs.outputs.targetBranch }}"
- persist-credentials: false
- if: steps.defaults.outputs.proceed == 'true'
- name: "Setup python"
uses: actions/setup-python@v2
with:
@@ -363,7 +370,7 @@ jobs:
run: ./scripts/ci/images/ci_push_production_images.sh
if: matrix.image-type == 'PROD' && steps.defaults.outputs.proceed == 'true'
- name: Update Github Checks for Building image with status
- uses: apache/airflow-checks-action@9f02872da71b6f558c6a6f190f925dde5e4d8798 # v1.1.0
+ uses: ./main-airflow/.github/actions/checks-action
if: always() && steps.defaults.outputs.proceed == 'true'
with:
token: ${{ secrets.GITHUB_TOKEN }}
@@ -384,8 +391,13 @@ jobs:
if: cancelled()
needs: [build-images]
steps:
+ - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+ uses: actions/checkout@v2
+ with:
+ persist-credentials: false
+ submodules: recursive
- name: "Canceling the 'CI Build' source workflow in case of failure!"
- uses: apache/airflow-cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738 # v4_7
+ uses: ./.github/actions/cancel-workflow-runs
with:
token: ${{ secrets.GITHUB_TOKEN }}
cancelMode: self
@@ -399,8 +411,13 @@ jobs:
if: failure()
needs: [build-images]
steps:
+ - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+ uses: actions/checkout@v2
+ with:
+ persist-credentials: false
+ submodules: recursive
- name: "Canceling the 'CI Build' source workflow in case of failure!"
- uses: apache/airflow-cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738 # v4_7
+ uses: ./.github/actions/cancel-workflow-runs
with:
token: ${{ secrets.GITHUB_TOKEN }}
cancelMode: self
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 728dfb0..e87564d 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -98,15 +98,16 @@ jobs:
pullRequestNumber: ${{ steps.source-run-info.outputs.pullRequestNumber }}
pullRequestLabels: ${{ steps.source-run-info.outputs.pullRequestLabels }}
steps:
- - name: "Get information about the PR"
- uses: apache/airflow-get-workflow-origin@588cc14f9f1cdf1b8be3db816855e96422204fec # v1_3
- id: source-run-info
- with:
- token: ${{ secrets.GITHUB_TOKEN }}
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v2
with:
persist-credentials: false
+ submodules: recursive
+ - name: "Get information about the PR"
+ uses: ./.github/actions/get-workflow-origin
+ id: source-run-info
+ with:
+ token: ${{ secrets.GITHUB_TOKEN }}
- name: >
Event: ${{ github.event_name }}
Repo: ${{ steps.source-run-info.outputs.sourceHeadRepo }}
@@ -296,6 +297,7 @@ jobs:
uses: actions/checkout@v2
with:
persist-credentials: false
+ submodules: recursive
- name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- name: "Build docs"
@@ -617,6 +619,11 @@ jobs:
- tests-mysql
- tests-quarantined
steps:
+ - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+ uses: actions/checkout@v2
+ with:
+ persist-credentials: false
+ submodules: recursive
- name: "Download all artifacts from the current build"
uses: actions/download-artifact@v2
with:
@@ -624,7 +631,7 @@ jobs:
- name: "Removes unnecessary artifacts"
run: ls ./coverage-files | grep -v coverage | xargs rm -rf
- name: "Upload all coverage reports to codecov"
- uses: apache/airflow-codecov-action@1fc7722ded4708880a5aea49f2bfafb9336f0c8d # v1.1.1
+ uses: ./.github/actions/codecov-action
with:
directory: "./coverage-files"
@@ -905,6 +912,7 @@ jobs:
uses: actions/checkout@v2
with:
persist-credentials: false
+ submodules: recursive
- name: "Set constraints branch name"
id: constraints-branch
run: ./scripts/ci/constraints/ci_branch_constraints.sh
@@ -921,7 +929,7 @@ jobs:
- name: "Commit changed constraint files for ${{needs.build-info.outputs.pythonVersions}}"
run: ./scripts/ci/constraints/ci_commit_constraints.sh
- name: "Push changes"
- uses: apache/airflow-github-push-action@40bf560936a8022e68a3c00e7d2abefaf01305a6 # v0.6.0
+ uses: ./.github/actions/github-push-action
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
branch: ${{ steps.constraints-branch.outputs.branch }}
diff --git a/.github/workflows/label_when_reviewed_workflow_run.yml b/.github/workflows/label_when_reviewed_workflow_run.yml
index 4400bae..1ed50dd 100644
--- a/.github/workflows/label_when_reviewed_workflow_run.yml
+++ b/.github/workflows/label_when_reviewed_workflow_run.yml
@@ -29,14 +29,19 @@ jobs:
outputs:
labelSet: ${{ steps.label-when-reviewed.outputs.labelSet }}
steps:
+ - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+ uses: actions/checkout@v2
+ with:
+ persist-credentials: false
+ submodules: recursive
- name: "Get information about the original trigger of the run"
- uses: apache/airflow-get-workflow-origin@588cc14f9f1cdf1b8be3db816855e96422204fec # v1_3
+ uses: ./.github/actions/get-workflow-origin
id: source-run-info
with:
token: ${{ secrets.GITHUB_TOKEN }}
sourceRunId: ${{ github.event.workflow_run.id }}
- name: Initiate Selective Build check
- uses: apache/airflow-checks-action@9f02872da71b6f558c6a6f190f925dde5e4d8798 # v1.1.0
+ uses: ./.github/actions/checks-action
id: selective-build-check
with:
token: ${{ secrets.GITHUB_TOKEN }}
@@ -49,10 +54,6 @@ jobs:
"Checking selective status of the build in
[the run](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})
"}
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v2
- with:
- persist-credentials: false
- name: >
Event: ${{ steps.source-run-info.outputs.sourceEvent }}
Repo: ${{ steps.source-run-info.outputs.sourceHeadRepo }}
@@ -76,6 +77,7 @@ jobs:
uses: actions/checkout@v2
with:
persist-credentials: false
+ submodules: recursive
- name: Selective checks
id: selective-checks
env:
@@ -91,7 +93,7 @@ jobs:
./scripts/ci/selective_ci_checks.sh
fi
- name: "Label when approved by committers for PRs that require full tests"
- uses: apache/airflow-label-when-approved@4c5190fec5661e98d83f50bbd4ef9ebb48bd1194 # v1.3
+ uses: ./.github/actions/label-when-approved-action
id: label-full-test-prs-when-approved-by-commiters
if: >
steps.selective-checks.outputs.run-tests == 'true' &&
@@ -107,7 +109,7 @@ jobs:
If they don't merge it quickly - please rebase it to the latest master at your convenience,
or amend the last commit of the PR, and push it with --force-with-lease.
- name: "Initiate GitHub Check forcing rerun of SH ${{ github.event.pull_request.head.sha }}"
- uses: apache/airflow-checks-action@9f02872da71b6f558c6a6f190f925dde5e4d8798 # v1.1.0
+ uses: ./.github/actions/checks-action
id: full-test-check
if: steps.label-full-test-prs-when-approved-by-commiters.outputs.labelSet == 'true'
with:
@@ -122,7 +124,7 @@ jobs:
[the run](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})
"}
- name: "Label when approved by committers for PRs that do not require full tests"
- uses: apache/airflow-label-when-approved@4c5190fec5661e98d83f50bbd4ef9ebb48bd1194 # v1.3
+ uses: ./.github/actions/label-when-approved-action
id: label-simple-test-prs-when-approved-by-commiters
if: >
steps.selective-checks.outputs.run-tests == 'true' &&
@@ -139,7 +141,7 @@ jobs:
'full tests needed'. Then you should rebase to the latest master or amend the last commit
of the PR, and push it with --force-with-lease.
- name: "Label when approved by committers for PRs that do not require tests at all"
- uses: apache/airflow-label-when-approved@4c5190fec5661e98d83f50bbd4ef9ebb48bd1194 # v1.3
+ uses: ./.github/actions/label-when-approved-action
id: label-no-test-prs-when-approved-by-commiters
if: steps.selective-checks.outputs.run-tests != 'true'
with:
@@ -153,7 +155,7 @@ jobs:
needed and add the 'full tests needed' label. Then you should rebase it to the latest master
or amend the last commit of the PR, and push it with --force-with-lease.
- name: Update Selective Build check
- uses: apache/airflow-checks-action@9f02872da71b6f558c6a6f190f925dde5e4d8798 # v1.1.0
+ uses: ./.github/actions/checks-action
if: always()
with:
token: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.gitmodules b/.gitmodules
new file mode 100644
index 0000000..1779ddd
--- /dev/null
+++ b/.gitmodules
@@ -0,0 +1,21 @@
+[submodule ".github/actions/get-workflow-origin"]
+ path = .github/actions/get-workflow-origin
+ url = https://github.com/potiuk/get-workflow-origin
+[submodule ".github/actions/cancel-workflow-runs"]
+ path = .github/actions/cancel-workflow-runs
+ url = https://github.com/potiuk/cancel-workflow-runs
+[submodule ".github/actions/checks-action"]
+ path = .github/actions/checks-action
+ url = https://github.com/LouisBrunner/checks-action
+[submodule ".github/actions/configure-aws-credentials"]
+ path = .github/actions/configure-aws-credentials
+ url = https://github.com/aws-actions/configure-aws-credentials
+[submodule ".github/actions/codecov-action"]
+ path = .github/actions/codecov-action
+ url = https://github.com/codecov/codecov-action
+[submodule ".github/actions/github-push-action"]
+ path = .github/actions/github-push-action
+ url = https://github.com/ad-m/github-push-action
+[submodule ".github/actions/label-when-approved-action"]
+ path = .github/actions/label-when-approved-action
+ url = https://github.com/TobKed/label-when-approved-action
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 2e6207d..a98b0ea 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -29,7 +29,7 @@ repos:
rev: v1.1.9
hooks:
- id: forbid-tabs
- exclude: ^docs/Makefile$|^clients/gen/go.sh
+ exclude: ^docs/Makefile$|^clients/gen/go.sh|\.gitmodules$
- id: insert-license
name: Add license for all SQL files
files: \.sql$