You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2022/07/01 13:11:20 UTC
[airflow] branch main updated: Better diagnostics in case someone modified the dependencies file (#24763)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 1b3905ef6e Better diagnostics in case someone modified the dependencies file (#24763)
1b3905ef6e is described below
commit 1b3905ef6eb5630e8d12975d9e91600ffb832471
Author: Jarek Potiuk <ja...@polidea.com>
AuthorDate: Fri Jul 1 15:11:10 2022 +0200
Better diagnostics in case someone modified the dependencies file (#24763)
The "provider_dependencies.json" file gets automatically generated
by pre-commit and in case it cannot be json-parsed, it will break
building the CI image. This is a bit chicken-egg because on
CI the image is needed to run pre-commits that could warn the
user this is the case (and the image fails build with a bit
cryptic message).
This PR improves the diagnostics:
* it runs the pre-commit check before image building which will
fix the generated file (and will let the build run - only to
fail at the latest static-checks step
* it prints warning to the user seeing pre-commit error to not
modify the file manually but let pre-commit do the job.
---
.github/workflows/build-images.yml | 22 ++++++++++++++++++++
.github/workflows/ci.yml | 24 ++++++++++++++++++++++
.../pre_commit_build_providers_dependencies.py | 4 ++++
3 files changed, 50 insertions(+)
diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml
index 202f25b0d3..a2218833e7 100644
--- a/.github/workflows/build-images.yml
+++ b/.github/workflows/build-images.yml
@@ -224,6 +224,17 @@ jobs:
- run: ./scripts/ci/install_breeze.sh
- name: "Free space"
run: breeze free-space
+ - name: Cache pre-commit envs
+ uses: actions/cache@v3
+ with:
+ path: ~/.cache/pre-commit
+ key: "pre-commit-${{steps.host-python-version.outputs.host-python-version}}-\
+${{ hashFiles('.pre-commit-config.yaml') }}"
+ restore-keys: pre-commit-${{steps.host-python-version.outputs.host-python-version}}
+ - name: "Regenerate dependencies in case they was modified manually so that we can build an image"
+ run: >
+ breeze static-checks --type update-providers-dependencies --all-files
+ --show-diff-on-failure --color always || true
- name: >
Build & Push AMD64 CI images ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
${{ needs.build-info.outputs.allPythonVersionsListAsString }}
@@ -316,6 +327,17 @@ jobs:
- run: ./scripts/ci/install_breeze.sh
- name: "Free space"
run: breeze free-space
+ - name: Cache pre-commit envs
+ uses: actions/cache@v3
+ with:
+ path: ~/.cache/pre-commit
+ key: "pre-commit-${{steps.host-python-version.outputs.host-python-version}}-\
+${{ hashFiles('.pre-commit-config.yaml') }}"
+ restore-keys: pre-commit-${{steps.host-python-version.outputs.host-python-version}}
+ - name: "Regenerate dependencies in case they was modified manually so that we can build an image"
+ run: >
+ breeze static-checks --type update-providers-dependencies --all-files
+ --show-diff-on-failure --color always || true
- name: >
Pull CI image for PROD build:
${{ needs.build-info.outputs.defaultPythonVersion }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }}
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index b3b3438601..41e894e5a5 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -337,6 +337,18 @@ jobs:
- name: "Free space"
run: breeze free-space
if: needs.build-info.outputs.inWorkflowBuild == 'true'
+ - name: Cache pre-commit envs
+ uses: actions/cache@v3
+ with:
+ path: ~/.cache/pre-commit
+ key: "pre-commit-${{steps.host-python-version.outputs.host-python-version}}-\
+${{ hashFiles('.pre-commit-config.yaml') }}"
+ restore-keys: pre-commit-${{steps.host-python-version.outputs.host-python-version}}
+ - name: "Regenerate dependencies in case they was modified manually so that we can build an image"
+ run: >
+ breeze static-checks --type update-providers-dependencies --all-files
+ --show-diff-on-failure --color always || true
+ if: needs.build-info.outputs.inWorkflowBuild == 'true'
- name: >
Build & Push CI images ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
${{ needs.build-info.outputs.allPythonVersionsListAsString }}
@@ -393,6 +405,18 @@ jobs:
- name: "Free space"
run: breeze free-space
if: needs.build-info.outputs.inWorkflowBuild == 'true'
+ - name: Cache pre-commit envs
+ uses: actions/cache@v3
+ with:
+ path: ~/.cache/pre-commit
+ key: "pre-commit-${{steps.host-python-version.outputs.host-python-version}}-\
+${{ hashFiles('.pre-commit-config.yaml') }}"
+ restore-keys: pre-commit-${{steps.host-python-version.outputs.host-python-version}}
+ - name: "Regenerate dependencies in case they was modified manually so that we can build an image"
+ run: >
+ breeze static-checks --type update-providers-dependencies --all-files
+ --show-diff-on-failure --color always || true
+ if: needs.build-info.outputs.inWorkflowBuild == 'true'
- name: >
Pull CI image for PROD build:
${{ needs.build-info.outputs.defaultPythonVersion }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }}"
diff --git a/scripts/ci/pre_commit/pre_commit_build_providers_dependencies.py b/scripts/ci/pre_commit/pre_commit_build_providers_dependencies.py
index af2471ad62..8ee81908f4 100755
--- a/scripts/ci/pre_commit/pre_commit_build_providers_dependencies.py
+++ b/scripts/ci/pre_commit/pre_commit_build_providers_dependencies.py
@@ -195,6 +195,10 @@ if __name__ == '__main__':
console.print()
sys.exit(1)
DEPENDENCIES_JSON_FILE_PATH.write_text(json.dumps(unique_sorted_dependencies, indent=2) + "\n")
+ console.print(
+ f"[yellow]If you see changes to the {DEPENDENCIES_JSON_FILE_PATH} file - "
+ f"do not modify the file manually. Let pre-commit do the job!"
+ )
console.print()
console.print("[green]Verification complete! Success!\n")
console.print(f"Written {DEPENDENCIES_JSON_FILE_PATH}")