You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ep...@apache.org on 2023/03/08 13:18:07 UTC
[airflow] 12/12: Make static checks generated file more stable accross the board (#29080)
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch v2-5-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 4e2af12f99dbb816d341751490921f3eca34b047
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Mon Jan 23 12:48:33 2023 +0100
Make static checks generated file more stable accross the board (#29080)
There were couple of problems with static checks generating source
files including generated stubs in the common.sql package:
* black formatting was implemented in multiple separate scripts
making it harded to fix problems in all of them
* generated stub files were not formatted with is_pyi=True and
black had no way to figure it out because it was working on strings
* black formatting was not consistently applied in all places
* EOL at the end of generated stub file was missing, leading to EOL
fixer adding them after generation leading to multiple pre-commit
passes needed
* there was (already unused) deprecated dev dict generator that used
its own black formatting.
There were also couple of problems with the files generated by
stubgen itself:
* Union was missing in the generated stubs (this is a known issue
with stubgen: https://github.com/python/mypy/issues/12929
* Intellij complained on Incomplete import from _typeshed missing
This PR fixes all the problems:
* black formatting is now consistenly extracted and applied everywhere
* when needed, is_pyi flag is passed to black so that it knows
that .pyi file is being fomratted
* EOL is added at the end of file when the file is generated
* Union is added to the generated stub
* noqa is added to _typeshed import
* the dict generator is removed
As the end result, generated stub files are fully importable
(no errors reported by IntelliJ IDE) and consistently formatted
every time.
(cherry picked from commit 129f0820cd03c721ebebe3461489f255bb9e752c)
---
.pre-commit-config.yaml | 21 +-
dev/deprecations/generate_deprecated_dicts.py | 217 ---------------------
dev/provider_packages/prepare_provider_packages.py | 18 +-
.../ci/pre_commit/common_precommit_black_utils.py | 44 +++++
scripts/ci/pre_commit/common_precommit_utils.py | 3 +-
.../pre_commit_check_pre_commit_hooks.py | 70 +++----
.../ci/pre_commit/pre_commit_compile_www_assets.py | 3 +-
scripts/ci/pre_commit/pre_commit_insert_extras.py | 4 +-
.../ci/pre_commit/pre_commit_local_yml_mounts.py | 18 +-
scripts/ci/pre_commit/pre_commit_mypy.py | 13 +-
10 files changed, 105 insertions(+), 306 deletions(-)
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 07e5d18d58..4d06c974dc 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -146,6 +146,13 @@ repos:
- --fuzzy-match-generates-todo
files: >
\.cfg$|\.conf$|\.ini$|\.ldif$|\.properties$|\.readthedocs$|\.service$|\.tf$|Dockerfile.*$
+ - repo: https://github.com/psf/black
+ rev: 22.12.0
+ hooks:
+ - id: black
+ name: Run black (python formatter)
+ args: [--config=./pyproject.toml]
+ exclude: ^airflow/_vendor/|^airflow/contrib/
- repo: local
hooks:
- id: update-black-version
@@ -167,15 +174,8 @@ repos:
additional_dependencies: ['ruff>=0.0.219']
files: \.pyi?$
exclude: ^airflow/_vendor/
- - repo: https://github.com/psf/black
- rev: 22.12.0
- hooks:
- - id: black
- name: Run black (python formatter)
- args: [--config=./pyproject.toml]
- exclude: ^airflow/_vendor/|^airflow/contrib/
- repo: https://github.com/asottile/blacken-docs
- rev: v1.12.1
+ rev: 1.13.0
hooks:
- id: blacken-docs
name: Run black on python code blocks in documentation files
@@ -230,7 +230,7 @@ repos:
files: ^chart/values\.schema\.json$|^chart/values_schema\.schema\.json$
pass_filenames: true
- repo: https://github.com/pre-commit/pygrep-hooks
- rev: v1.9.0
+ rev: v1.10.0
hooks:
- id: rst-backticks
name: Check if RST files use double backticks for code
@@ -239,7 +239,7 @@ repos:
name: Check if there are no deprecate log warn
exclude: ^airflow/_vendor/
- repo: https://github.com/adrienverge/yamllint
- rev: v1.26.3
+ rev: v1.29.0
hooks:
- id: yamllint
name: Check YAML files with yamllint
@@ -344,6 +344,7 @@ repos:
language: python
files: ^setup\.py$|^INSTALL$|^CONTRIBUTING\.rst$
pass_filenames: false
+ additional_dependencies: ['rich>=12.4.4']
- id: check-extras-order
name: Check order of extras in Dockerfile
entry: ./scripts/ci/pre_commit/pre_commit_check_order_dockerfile_extras.py
diff --git a/dev/deprecations/generate_deprecated_dicts.py b/dev/deprecations/generate_deprecated_dicts.py
deleted file mode 100644
index b705fee48b..0000000000
--- a/dev/deprecations/generate_deprecated_dicts.py
+++ /dev/null
@@ -1,217 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-from __future__ import annotations
-
-import ast
-import os
-from collections import defaultdict
-from functools import lru_cache
-from pathlib import Path
-from typing import NamedTuple
-
-from jinja2 import BaseLoader, Environment
-from rich.console import Console
-
-if __name__ not in ("__main__", "__mp_main__"):
- raise SystemExit(
- "This file is intended to be executed as an executable program. You cannot use it as a module_path."
- f"To run this script, run the ./{__file__} command [FILE] ..."
- )
-
-AIRFLOW_SOURCES_ROOT = Path(__file__).parents[2].resolve()
-CONTRIB_DIR = AIRFLOW_SOURCES_ROOT / "airflow" / "contrib"
-
-
-@lru_cache(maxsize=None)
-def black_mode():
- from black import Mode, parse_pyproject_toml, target_version_option_callback
-
- config = parse_pyproject_toml(os.path.join(AIRFLOW_SOURCES_ROOT, "pyproject.toml"))
-
- target_versions = set(
- target_version_option_callback(None, None, tuple(config.get("target_version", ()))),
- )
-
- return Mode(
- target_versions=target_versions,
- line_length=config.get("line_length", Mode.line_length),
- is_pyi=bool(config.get("is_pyi", Mode.is_pyi)),
- string_normalization=not bool(config.get("skip_string_normalization", not Mode.string_normalization)),
- experimental_string_processing=bool(
- config.get("experimental_string_processing", Mode.experimental_string_processing)
- ),
- )
-
-
-def black_format(content) -> str:
- from black import format_str
-
- return format_str(content, mode=black_mode())
-
-
-class Import(NamedTuple):
- module_path: str
- name: str
- alias: str
-
-
-class ImportedClass(NamedTuple):
- module_path: str
- name: str
-
-
-def get_imports(path: Path):
- root = ast.parse(path.read_text())
- imports: dict[str, ImportedClass] = {}
- for node in ast.iter_child_nodes(root):
- if isinstance(node, ast.Import):
- module_array: list[str] = []
- elif isinstance(node, ast.ImportFrom) and node.module:
- module_array = node.module.split(".")
- elif isinstance(node, ast.ClassDef):
- for base in node.bases:
- res = imports.get(base.id) # type: ignore[attr-defined]
- if res:
- yield Import(module_path=res.module_path, name=res.name, alias=node.name)
- continue
- else:
- continue
- for n in node.names: # type: ignore[attr-defined]
- imported_as = n.asname if n.asname else n.name
- module_path = ".".join(module_array)
- imports[imported_as] = ImportedClass(module_path=module_path, name=n.name)
- yield Import(module_path, n.name, imported_as)
-
-
-DEPRECATED_CLASSES_TEMPLATE = """
-__deprecated_classes = {
-{%- for module_path, package_imports in package_imports.items() %}
- '{{module_path}}': {
-{%- for import_item in package_imports %}
- '{{import_item.alias}}': '{{import_item.module_path}}.{{import_item.name}}',
-{%- endfor %}
- },
-{%- endfor %}
-}
-"""
-
-DEPRECATED_MODULES = [
- "airflow/hooks/base_hook.py",
- "airflow/hooks/dbapi_hook.py",
- "airflow/hooks/docker_hook.py",
- "airflow/hooks/druid_hook.py",
- "airflow/hooks/hdfs_hook.py",
- "airflow/hooks/hive_hooks.py",
- "airflow/hooks/http_hook.py",
- "airflow/hooks/jdbc_hook.py",
- "airflow/hooks/mssql_hook.py",
- "airflow/hooks/mysql_hook.py",
- "airflow/hooks/oracle_hook.py",
- "airflow/hooks/pig_hook.py",
- "airflow/hooks/postgres_hook.py",
- "airflow/hooks/presto_hook.py",
- "airflow/hooks/S3_hook.py",
- "airflow/hooks/samba_hook.py",
- "airflow/hooks/slack_hook.py",
- "airflow/hooks/sqlite_hook.py",
- "airflow/hooks/webhdfs_hook.py",
- "airflow/hooks/zendesk_hook.py",
- "airflow/operators/bash_operator.py",
- "airflow/operators/branch_operator.py",
- "airflow/operators/check_operator.py",
- "airflow/operators/dagrun_operator.py",
- "airflow/operators/docker_operator.py",
- "airflow/operators/druid_check_operator.py",
- "airflow/operators/dummy.py",
- "airflow/operators/dummy_operator.py",
- "airflow/operators/email_operator.py",
- "airflow/operators/gcs_to_s3.py",
- "airflow/operators/google_api_to_s3_transfer.py",
- "airflow/operators/hive_operator.py",
- "airflow/operators/hive_stats_operator.py",
- "airflow/operators/hive_to_druid.py",
- "airflow/operators/hive_to_mysql.py",
- "airflow/operators/hive_to_samba_operator.py",
- "airflow/operators/http_operator.py",
- "airflow/operators/jdbc_operator.py",
- "airflow/operators/latest_only_operator.py",
- "airflow/operators/mssql_operator.py",
- "airflow/operators/mssql_to_hive.py",
- "airflow/operators/mysql_operator.py",
- "airflow/operators/mysql_to_hive.py",
- "airflow/operators/oracle_operator.py",
- "airflow/operators/papermill_operator.py",
- "airflow/operators/pig_operator.py",
- "airflow/operators/postgres_operator.py",
- "airflow/operators/presto_check_operator.py",
- "airflow/operators/presto_to_mysql.py",
- "airflow/operators/python_operator.py",
- "airflow/operators/redshift_to_s3_operator.py",
- "airflow/operators/s3_file_transform_operator.py",
- "airflow/operators/s3_to_hive_operator.py",
- "airflow/operators/s3_to_redshift_operator.py",
- "airflow/operators/slack_operator.py",
- "airflow/operators/sql.py",
- "airflow/operators/sql_branch_operator.py",
- "airflow/operators/sqlite_operator.py",
- "airflow/operators/subdag_operator.py",
- "airflow/sensors/base_sensor_operator.py",
- "airflow/sensors/date_time_sensor.py",
- "airflow/sensors/external_task_sensor.py",
- "airflow/sensors/hdfs_sensor.py",
- "airflow/sensors/hive_partition_sensor.py",
- "airflow/sensors/http_sensor.py",
- "airflow/sensors/metastore_partition_sensor.py",
- "airflow/sensors/named_hive_partition_sensor.py",
- "airflow/sensors/s3_key_sensor.py",
- "airflow/sensors/sql.py",
- "airflow/sensors/sql_sensor.py",
- "airflow/sensors/time_delta_sensor.py",
- "airflow/sensors/web_hdfs_sensor.py",
- "airflow/utils/log/cloudwatch_task_handler.py",
- "airflow/utils/log/es_task_handler.py",
- "airflow/utils/log/gcs_task_handler.py",
- "airflow/utils/log/s3_task_handler.py",
- "airflow/utils/log/stackdriver_task_handler.py",
- "airflow/utils/log/wasb_task_handler.py",
-]
-
-CONTRIB_FILES = (AIRFLOW_SOURCES_ROOT / "airflow" / "contrib").rglob("*.py")
-
-
-if __name__ == "__main__":
- console = Console(color_system="standard", width=300)
- all_deprecated_imports: dict[str, dict[str, list[Import]]] = defaultdict(lambda: defaultdict(list))
- # delete = True
- delete = False
- # for file in DEPRECATED_MODULES:
- for file in CONTRIB_FILES:
- file_path = AIRFLOW_SOURCES_ROOT / file
- if not file_path.exists() or file.name == "__init__.py":
- continue
- original_module = os.fspath(file_path.parent.relative_to(AIRFLOW_SOURCES_ROOT)).replace(os.sep, ".")
- for _import in get_imports(file_path):
- module_name = file_path.name[: -len(".py")]
- if _import.name not in ["warnings", "RemovedInAirflow3Warning"]:
- all_deprecated_imports[original_module][module_name].append(_import)
- if delete:
- file_path.unlink()
-
- for module_path, package_imports in all_deprecated_imports.items():
- console.print(f"[yellow]Import dictionary for {module_path}:\n")
- template = Environment(loader=BaseLoader()).from_string(DEPRECATED_CLASSES_TEMPLATE)
- print(black_format(template.render(package_imports=dict(sorted(package_imports.items())))))
diff --git a/dev/provider_packages/prepare_provider_packages.py b/dev/provider_packages/prepare_provider_packages.py
index 96c00dffea..ac27bcafa1 100755
--- a/dev/provider_packages/prepare_provider_packages.py
+++ b/dev/provider_packages/prepare_provider_packages.py
@@ -46,6 +46,7 @@ from typing import Any, Generator, Iterable, NamedTuple
import jsonschema
import rich_click as click
import semver as semver
+from black import Mode, TargetVersion, format_str, parse_pyproject_toml
from packaging.version import Version
from rich.console import Console
from rich.syntax import Syntax
@@ -1393,29 +1394,16 @@ def update_commits_rst(
@lru_cache(maxsize=None)
-def black_mode():
- from black import Mode, parse_pyproject_toml, target_version_option_callback
-
+def black_mode() -> Mode:
config = parse_pyproject_toml(os.path.join(AIRFLOW_SOURCES_ROOT_PATH, "pyproject.toml"))
-
- target_versions = set(
- target_version_option_callback(None, None, tuple(config.get("target_version", ()))),
- )
-
+ target_versions = {TargetVersion[val.upper()] for val in config.get("target_version", ())}
return Mode(
target_versions=target_versions,
line_length=config.get("line_length", Mode.line_length),
- is_pyi=bool(config.get("is_pyi", Mode.is_pyi)),
- string_normalization=not bool(config.get("skip_string_normalization", not Mode.string_normalization)),
- experimental_string_processing=bool(
- config.get("experimental_string_processing", Mode.experimental_string_processing)
- ),
)
def black_format(content) -> str:
- from black import format_str
-
return format_str(content, mode=black_mode())
diff --git a/scripts/ci/pre_commit/common_precommit_black_utils.py b/scripts/ci/pre_commit/common_precommit_black_utils.py
new file mode 100644
index 0000000000..c9d0f77122
--- /dev/null
+++ b/scripts/ci/pre_commit/common_precommit_black_utils.py
@@ -0,0 +1,44 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import os
+import sys
+from functools import lru_cache
+from pathlib import Path
+
+from black import Mode, TargetVersion, format_str, parse_pyproject_toml
+
+sys.path.insert(0, str(Path(__file__).parent.resolve())) # make sure common_precommit_utils is imported
+
+from common_precommit_utils import AIRFLOW_BREEZE_SOURCES_PATH # isort: skip # noqa E402
+
+
+@lru_cache(maxsize=None)
+def black_mode(is_pyi: bool = Mode.is_pyi) -> Mode:
+ config = parse_pyproject_toml(os.fspath(AIRFLOW_BREEZE_SOURCES_PATH / "pyproject.toml"))
+ target_versions = {TargetVersion[val.upper()] for val in config.get("target_version", ())}
+
+ return Mode(
+ target_versions=target_versions,
+ line_length=config.get("line_length", Mode.line_length),
+ is_pyi=is_pyi,
+ )
+
+
+def black_format(content: str, is_pyi: bool = Mode.is_pyi) -> str:
+ return format_str(content, mode=black_mode(is_pyi=is_pyi))
diff --git a/scripts/ci/pre_commit/common_precommit_utils.py b/scripts/ci/pre_commit/common_precommit_utils.py
index 3dc1fceba5..29109a4c34 100644
--- a/scripts/ci/pre_commit/common_precommit_utils.py
+++ b/scripts/ci/pre_commit/common_precommit_utils.py
@@ -22,7 +22,8 @@ import os
import re
from pathlib import Path
-AIRFLOW_SOURCES_ROOT = Path(__file__).parents[3].resolve()
+AIRFLOW_SOURCES_ROOT_PATH = Path(__file__).parents[3].resolve()
+AIRFLOW_BREEZE_SOURCES_PATH = AIRFLOW_SOURCES_ROOT_PATH / "dev" / "breeze"
def read_airflow_version() -> str:
diff --git a/scripts/ci/pre_commit/pre_commit_check_pre_commit_hooks.py b/scripts/ci/pre_commit/pre_commit_check_pre_commit_hooks.py
index d6e32a0937..1c98d5cb41 100755
--- a/scripts/ci/pre_commit/pre_commit_check_pre_commit_hooks.py
+++ b/scripts/ci/pre_commit/pre_commit_check_pre_commit_hooks.py
@@ -26,22 +26,24 @@ import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).parent.resolve())) # make sure common_precommit_utils is imported
+from common_precommit_utils import ( # isort: skip # noqa: E402
+ AIRFLOW_BREEZE_SOURCES_PATH,
+ AIRFLOW_SOURCES_ROOT_PATH,
+ insert_documentation,
+)
+from common_precommit_black_utils import black_format # isort: skip # noqa E402
from collections import defaultdict # noqa: E402
-from functools import lru_cache # noqa: E402
from typing import Any # noqa: E402
import yaml # noqa: E402
-from common_precommit_utils import insert_documentation # noqa: E402
from rich.console import Console # noqa: E402
from tabulate import tabulate # noqa: E402
console = Console(width=400, color_system="standard")
-AIRFLOW_SOURCES_PATH = Path(__file__).parents[3].resolve()
-AIRFLOW_BREEZE_SOURCES_PATH = AIRFLOW_SOURCES_PATH / "dev" / "breeze"
PRE_COMMIT_IDS_PATH = AIRFLOW_BREEZE_SOURCES_PATH / "src" / "airflow_breeze" / "pre_commit_ids.py"
-PRE_COMMIT_YAML_FILE = AIRFLOW_SOURCES_PATH / ".pre-commit-config.yaml"
+PRE_COMMIT_YAML_FILE = AIRFLOW_SOURCES_ROOT_PATH / ".pre-commit-config.yaml"
def get_errors_and_hooks(content: Any, max_length: int) -> tuple[list[str], dict[str, list[str]], list[str]]:
@@ -75,6 +77,22 @@ def get_errors_and_hooks(content: Any, max_length: int) -> tuple[list[str], dict
return errors, hooks, image_hooks
+def prepare_pre_commit_ids_py_file(pre_commit_ids):
+ PRE_COMMIT_IDS_PATH.write_text(
+ black_format(
+ content=render_template(
+ searchpath=AIRFLOW_BREEZE_SOURCES_PATH / "src" / "airflow_breeze",
+ template_name="pre_commit_ids",
+ context={"PRE_COMMIT_IDS": pre_commit_ids},
+ extension=".py",
+ autoescape=False,
+ keep_trailing_newline=True,
+ ),
+ is_pyi=False,
+ )
+ )
+
+
def render_template(
searchpath: Path,
template_name: str,
@@ -107,46 +125,6 @@ def render_template(
return content
-@lru_cache(maxsize=None)
-def black_mode():
- from black import Mode, parse_pyproject_toml, target_version_option_callback
-
- config = parse_pyproject_toml(AIRFLOW_BREEZE_SOURCES_PATH / "pyproject.toml")
-
- target_versions = set(
- target_version_option_callback(None, None, tuple(config.get("target_version", ()))),
- )
-
- return Mode(
- target_versions=target_versions,
- line_length=config.get("line_length", Mode.line_length),
- is_pyi=config.get("is_pyi", False),
- string_normalization=not config.get("skip_string_normalization", False),
- preview=config.get("preview", False),
- )
-
-
-def black_format(content) -> str:
- from black import format_str
-
- return format_str(content, mode=black_mode())
-
-
-def prepare_pre_commit_ids_py_file(pre_commit_ids):
- PRE_COMMIT_IDS_PATH.write_text(
- black_format(
- render_template(
- searchpath=AIRFLOW_BREEZE_SOURCES_PATH / "src" / "airflow_breeze",
- template_name="pre_commit_ids",
- context={"PRE_COMMIT_IDS": pre_commit_ids},
- extension=".py",
- autoescape=False,
- keep_trailing_newline=True,
- )
- )
- )
-
-
def update_static_checks_array(hooks: dict[str, list[str]], image_hooks: list[str]):
rows = []
hook_ids = list(hooks.keys())
@@ -159,7 +137,7 @@ def update_static_checks_array(hooks: dict[str, list[str]], image_hooks: list[st
rows.append((hook_id, formatted_hook_description, " * " if hook_id in image_hooks else " "))
formatted_table = "\n" + tabulate(rows, tablefmt="grid", headers=("ID", "Description", "Image")) + "\n\n"
insert_documentation(
- file_path=AIRFLOW_SOURCES_PATH / "STATIC_CODE_CHECKS.rst",
+ file_path=AIRFLOW_SOURCES_ROOT_PATH / "STATIC_CODE_CHECKS.rst",
content=formatted_table.splitlines(keepends=True),
header=" .. BEGIN AUTO-GENERATED STATIC CHECK LIST",
footer=" .. END AUTO-GENERATED STATIC CHECK LIST",
diff --git a/scripts/ci/pre_commit/pre_commit_compile_www_assets.py b/scripts/ci/pre_commit/pre_commit_compile_www_assets.py
index 27975f6b8c..4733a1460b 100755
--- a/scripts/ci/pre_commit/pre_commit_compile_www_assets.py
+++ b/scripts/ci/pre_commit/pre_commit_compile_www_assets.py
@@ -23,7 +23,8 @@ import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).parent.resolve())) # make sure common_precommit_utils is imported
-from common_precommit_utils import get_directory_hash # isort: skip # noqa
+from common_precommit_utils import get_directory_hash # isort: skip # noqa E402
+from common_precommit_black_utils import black_format # isort: skip # noqa E402
AIRFLOW_SOURCES_PATH = Path(__file__).parents[3].resolve()
WWW_HASH_FILE = AIRFLOW_SOURCES_PATH / ".build" / "www" / "hash.txt"
diff --git a/scripts/ci/pre_commit/pre_commit_insert_extras.py b/scripts/ci/pre_commit/pre_commit_insert_extras.py
index fac926f611..3e08bd674d 100755
--- a/scripts/ci/pre_commit/pre_commit_insert_extras.py
+++ b/scripts/ci/pre_commit/pre_commit_insert_extras.py
@@ -27,8 +27,8 @@ sys.path.insert(0, str(Path(__file__).parent.resolve())) # make sure common_pre
sys.path.insert(0, str(AIRFLOW_SOURCES_DIR)) # make sure setup is imported from Airflow
# flake8: noqa: F401
-from common_precommit_utils import insert_documentation # isort: skip
-from setup import EXTRAS_DEPENDENCIES # isort:skip
+from common_precommit_utils import insert_documentation # isort: skip # noqa E402
+from setup import EXTRAS_DEPENDENCIES # isort:skip # noqa
sys.path.append(str(AIRFLOW_SOURCES_DIR))
diff --git a/scripts/ci/pre_commit/pre_commit_local_yml_mounts.py b/scripts/ci/pre_commit/pre_commit_local_yml_mounts.py
index 0f9f954959..6efba5a6aa 100755
--- a/scripts/ci/pre_commit/pre_commit_local_yml_mounts.py
+++ b/scripts/ci/pre_commit/pre_commit_local_yml_mounts.py
@@ -20,18 +20,20 @@ from __future__ import annotations
import sys
from pathlib import Path
-AIRFLOW_SOURCES_DIR = Path(__file__).parents[3].resolve()
-
sys.path.insert(0, str(Path(__file__).parent.resolve())) # make sure common_precommit_utils is imported
-sys.path.insert(0, str(AIRFLOW_SOURCES_DIR)) # make sure setup is imported from Airflow
+
+from common_precommit_utils import AIRFLOW_SOURCES_ROOT_PATH # isort: skip # noqa E402
+
+sys.path.insert(0, str(AIRFLOW_SOURCES_ROOT_PATH)) # make sure setup is imported from Airflow
sys.path.insert(
- 0, str(AIRFLOW_SOURCES_DIR / "dev" / "breeze" / "src")
+ 0, str(AIRFLOW_SOURCES_ROOT_PATH / "dev" / "breeze" / "src")
) # make sure setup is imported from Airflow
# flake8: noqa: F401
+from airflow_breeze.utils.docker_command_utils import VOLUMES_FOR_SELECTED_MOUNTS # isort: skip # noqa E402
-from common_precommit_utils import insert_documentation # isort: skip
+from common_precommit_utils import insert_documentation # isort: skip # noqa E402
-sys.path.append(str(AIRFLOW_SOURCES_DIR))
+sys.path.append(str(AIRFLOW_SOURCES_ROOT_PATH))
MOUNTS_HEADER = (
" # START automatically generated volumes from "
@@ -43,9 +45,7 @@ MOUNTS_FOOTER = (
)
if __name__ == "__main__":
- from airflow_breeze.utils.docker_command_utils import VOLUMES_FOR_SELECTED_MOUNTS
-
- local_mount_file_path = AIRFLOW_SOURCES_DIR / "scripts" / "ci" / "docker-compose" / "local.yml"
+ local_mount_file_path = AIRFLOW_SOURCES_ROOT_PATH / "scripts" / "ci" / "docker-compose" / "local.yml"
PREFIX = " "
volumes = []
for (src, dest) in VOLUMES_FOR_SELECTED_MOUNTS:
diff --git a/scripts/ci/pre_commit/pre_commit_mypy.py b/scripts/ci/pre_commit/pre_commit_mypy.py
index f6a08e8815..5c9de455b9 100755
--- a/scripts/ci/pre_commit/pre_commit_mypy.py
+++ b/scripts/ci/pre_commit/pre_commit_mypy.py
@@ -37,11 +37,14 @@ if __name__ == "__main__":
from common_precommit_utils import filter_out_providers_on_non_main_branch
sys.path.insert(0, str(AIRFLOW_SOURCES / "dev" / "breeze" / "src"))
- from airflow_breeze.global_constants import MOUNT_SELECTED
- from airflow_breeze.utils.console import get_console
- from airflow_breeze.utils.docker_command_utils import get_extra_docker_flags
- from airflow_breeze.utils.path_utils import create_mypy_volume_if_needed
- from airflow_breeze.utils.run_utils import get_ci_image_for_pre_commits, run_command
+ from airflow_breeze.global_constants import MOUNT_SELECTED # isort: skip
+ from airflow_breeze.utils.console import get_console # isort: skip
+ from airflow_breeze.utils.docker_command_utils import get_extra_docker_flags # isort: skip
+ from airflow_breeze.utils.path_utils import create_mypy_volume_if_needed # isort: skip
+ from airflow_breeze.utils.run_utils import (
+ get_ci_image_for_pre_commits,
+ run_command,
+ )
files_to_test = filter_out_providers_on_non_main_branch(sys.argv[1:])
if files_to_test == ["--namespace-packages"]: