You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ep...@apache.org on 2022/04/08 17:21:09 UTC
[airflow] branch main updated: Add `pre-commit` to check that `REVISION_HEADS_MAP` is up-to-date (#22860)
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 09e6f072c1 Add `pre-commit` to check that `REVISION_HEADS_MAP` is up-to-date (#22860)
09e6f072c1 is described below
commit 09e6f072c1a06594c4d2e8013e810e7ddd114fba
Author: Ephraim Anierobi <sp...@gmail.com>
AuthorDate: Fri Apr 8 18:20:54 2022 +0100
Add `pre-commit` to check that `REVISION_HEADS_MAP` is up-to-date (#22860)
This PR adds a `pre-commit` to make sure that the `REVISION_HEADS_MAP` is
up-to-date in any release. Since this feature will be out in 2.3.0 and is very important
to update it for releases, not just main, this would be helpful
---
.pre-commit-config.yaml | 5 ++
BREEZE.rst | 33 ++++++------
STATIC_CODE_CHECKS.rst | 2 +
breeze-complete | 1 +
dev/breeze/src/airflow_breeze/pre_commit_ids.py | 1 +
.../ci/pre_commit/pre_commit_version_heads_map.py | 62 ++++++++++++++++++++++
6 files changed, 88 insertions(+), 16 deletions(-)
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index c256c76036..3de7b7ad2b 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -364,6 +364,11 @@ repos:
files: ^scripts/ci/pre_commit/supported_versions\.py$|^README\.md$|^docs/apache-airflow/supported-versions\.rst$
pass_filenames: false
additional_dependencies: ['tabulate']
+ - id: check-revision-heads-map
+ name: Check that the REVISION_HEADS_MAP is up-to-date
+ language: python
+ entry: ./scripts/ci/pre_commit/pre_commit_version_heads_map.py
+ pass_filenames: false
- id: update-version
name: Update version to the latest version in the documentation
entry: ./scripts/ci/pre_commit/pre_commit_update_versions.py
diff --git a/BREEZE.rst b/BREEZE.rst
index f734d821d9..03608ee6f6 100644
--- a/BREEZE.rst
+++ b/BREEZE.rst
@@ -2278,22 +2278,23 @@ This is the current syntax for `./breeze <./breeze>`_:
build-providers-dependencies chart-schema-lint capitalized-breeze
changelog-duplicates check-2-1-compatibility check-apache-license
check-builtin-literals check-executables-have-shebangs check-extras-order
- check-hooks-apply check-integrations check-merge-conflict check-xml
- check-system-tests daysago-import-check debug-statements detect-private-key
- docstring-params doctoc dont-use-safe-filter end-of-file-fixer fix-encoding-pragma
- flake8 flynt codespell forbid-tabs helm-lint identity incorrect-use-of-LoggingMixin
- insert-license inline-dockerfile-scripts isort json-schema language-matters
- lint-dockerfile lint-openapi markdownlint mermaid migration-reference
- mixed-line-ending mypy mypy-helm no-providers-in-core-examples no-relative-imports
- persist-credentials-disabled pre-commit-descriptions pre-commit-hook-names
- pretty-format-json provide-create-sessions providers-changelogs providers-init-file
- providers-subpackages-init-file provider-yamls pydevd pydocstyle python-no-log-warn
- pyupgrade restrict-start_date rst-backticks setup-order setup-extra-packages
- shellcheck sort-in-the-wild sort-spelling-wordlist stylelint trailing-whitespace
- ui-lint update-breeze-config-hash update-breeze-file update-extras
- update-local-yml-file update-setup-cfg-file update-supported-versions
- update-versions vendor-k8s-json-schema verify-db-migrations-documented version-sync
- www-lint yamllint yesqa
+ check-hooks-apply check-integrations check-merge-conflict check-revision-heads-map
+ check-xml check-system-tests daysago-import-check debug-statements
+ detect-private-key docstring-params doctoc dont-use-safe-filter end-of-file-fixer
+ fix-encoding-pragma flake8 flynt codespell forbid-tabs helm-lint identity
+ incorrect-use-of-LoggingMixin insert-license inline-dockerfile-scripts isort
+ json-schema language-matters lint-dockerfile lint-openapi markdownlint mermaid
+ migration-reference mixed-line-ending mypy mypy-helm no-providers-in-core-examples
+ no-relative-imports persist-credentials-disabled pre-commit-descriptions
+ pre-commit-hook-names pretty-format-json provide-create-sessions
+ providers-changelogs providers-init-file providers-subpackages-init-file
+ provider-yamls pydevd pydocstyle python-no-log-warn pyupgrade restrict-start_date
+ rst-backticks setup-order setup-extra-packages shellcheck sort-in-the-wild
+ sort-spelling-wordlist stylelint trailing-whitespace ui-lint
+ update-breeze-config-hash update-breeze-file update-extras update-local-yml-file
+ update-setup-cfg-file update-supported-versions update-versions
+ vendor-k8s-json-schema verify-db-migrations-documented version-sync www-lint
+ yamllint yesqa
You can pass extra arguments including options to the pre-commit framework as
<EXTRA_ARGS> passed after --. For example:
diff --git a/STATIC_CODE_CHECKS.rst b/STATIC_CODE_CHECKS.rst
index 78b0202881..2fc0414345 100644
--- a/STATIC_CODE_CHECKS.rst
+++ b/STATIC_CODE_CHECKS.rst
@@ -165,6 +165,8 @@ require Breeze Docker images to be installed locally.
``check-integrations`` Checks if integration list is synchronized in code
------------------------------------ ---------------------------------------------------------------- ------------
``check-merge-conflicts`` Checks that merge conflicts are not being committed
+------------------------------------------------------------------------------------------------------------------
+``check-revision-heads-map`` Checks that REVISION_HEADS_MAP is up-to-date
------------------------------------ ---------------------------------------------------------------- ------------
``check-xml`` Checks XML files with xmllint
------------------------------------ ---------------------------------------------------------------- ------------
diff --git a/breeze-complete b/breeze-complete
index 77f6b6b6b3..b8e032aa37 100644
--- a/breeze-complete
+++ b/breeze-complete
@@ -98,6 +98,7 @@ check-extras-order
check-hooks-apply
check-integrations
check-merge-conflict
+check-revision-heads-map
check-xml
check-system-tests
daysago-import-check
diff --git a/dev/breeze/src/airflow_breeze/pre_commit_ids.py b/dev/breeze/src/airflow_breeze/pre_commit_ids.py
index 45957678f8..f9a9aaadbf 100644
--- a/dev/breeze/src/airflow_breeze/pre_commit_ids.py
+++ b/dev/breeze/src/airflow_breeze/pre_commit_ids.py
@@ -41,6 +41,7 @@ PRE_COMMIT_LIST = [
'check-hooks-apply',
'check-integrations',
'check-merge-conflict',
+ 'check-revision-heads-map',
'check-system-tests',
'check-xml',
'codespell',
diff --git a/scripts/ci/pre_commit/pre_commit_version_heads_map.py b/scripts/ci/pre_commit/pre_commit_version_heads_map.py
new file mode 100755
index 0000000000..7ba91ead0e
--- /dev/null
+++ b/scripts/ci/pre_commit/pre_commit_version_heads_map.py
@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import ast
+import sys
+from pathlib import Path
+
+PROJECT_SOURCE_ROOT_DIR = Path(__file__).resolve().parent.parent.parent.parent
+
+DB_FILE = PROJECT_SOURCE_ROOT_DIR / "airflow" / "utils" / "db.py"
+
+SETUP_FILE = PROJECT_SOURCE_ROOT_DIR / "setup.py"
+
+
+def read_revision_heads_map():
+ revision_heads_map_ast_obj = ast.parse(open(DB_FILE).read())
+
+ revision_heads_map_ast = [
+ a
+ for a in revision_heads_map_ast_obj.body
+ if isinstance(a, ast.Assign) and a.targets[0].id == "REVISION_HEADS_MAP"
+ ][0]
+
+ revision_heads_map = ast.literal_eval(revision_heads_map_ast.value)
+
+ return revision_heads_map.keys()
+
+
+def read_current_airflow_version():
+
+ ast_obj = ast.parse(open(SETUP_FILE).read())
+ assignments = [a for a in ast_obj.body if isinstance(a, ast.Assign)][:10]
+
+ version = [x for x in assignments if x.targets[0].id == "version"][0]
+
+ return ast.literal_eval(version.value)
+
+
+if __name__ == '__main__':
+ versions = read_revision_heads_map()
+ airflow_version = read_current_airflow_version()
+ if 'dev' not in airflow_version and airflow_version not in versions:
+ print("Current airflow version is not in the REVISION_HEADS_MAP")
+ print("Current airflow version:", airflow_version)
+ print("Please add the version to the REVISION_HEADS_MAP at:", DB_FILE)
+ sys.exit(3)