You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ka...@apache.org on 2025/04/07 15:13:10 UTC
(airflow) branch mik-laj-imports created (now cab7c37cf36)
This is an automated email from the ASF dual-hosted git repository.
kamilbregula pushed a change to branch mik-laj-imports
in repository https://gitbox.apache.org/repos/asf/airflow.git
at cab7c37cf36 Unify pre-coommit hooks for imports
This branch includes the following new commits:
new cab7c37cf36 Unify pre-coommit hooks for imports
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(airflow) 01/01: Unify pre-coommit hooks for imports
Posted by ka...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kamilbregula pushed a commit to branch mik-laj-imports
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit cab7c37cf36b5b4de80d3d693704f1853e3fe610
Author: Kamil Bregula <dz...@gmail.com>
AuthorDate: Mon Apr 7 15:12:50 2025 +0000
Unify pre-coommit hooks for imports
---
.pre-commit-config.yaml | 22 +++--
scripts/ci/pre_commit/check_airflow_imports.py | 95 ++++++++++++++++++++++
.../ci/pre_commit/check_airflow_k8s_not_used.py | 81 ------------------
.../check_cncf_k8s_used_for_k8s_executor_only.py | 81 ------------------
.../check_common_compat_used_for_openlineage.py | 88 --------------------
.../check_get_lineage_collector_providers.py | 83 -------------------
scripts/ci/pre_commit/common_precommit_utils.py | 39 +++++++++
.../ci/pre_commit/update_providers_dependencies.py | 42 +---------
8 files changed, 153 insertions(+), 378 deletions(-)
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 906cfa1cf74..67b2ee4eb01 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -402,15 +402,22 @@ repos:
files: ^airflow-core/src/airflow/.*\.py$
require_serial: true
exclude: ^airflow-core/src/airflow/kubernetes/
- entry: ./scripts/ci/pre_commit/check_airflow_k8s_not_used.py
+ entry: ./scripts/ci/pre_commit/check_airflow_imports.py
+ --pattern '^airflow\.kubernetes'
+ --message "You should only import kubernetes code from `airflow.providers.cncf.kubernetes`."
additional_dependencies: ['rich>=12.4.4']
- id: check-common-compat-used-for-openlineage
name: Check common.compat is used for OL deprecated classes
language: python
files: ^airflow-core/src/airflow/.*\.py$
require_serial: true
- exclude: ^airflow-core/src/airflow/openlineage/
- entry: ./scripts/ci/pre_commit/check_common_compat_used_for_openlineage.py
+ exclude: >
+ (?x)
+ ^airflow-core/src/airflow/openlineage/|
+ ^airflow/providers/common/compat/openlineage/facet.py$
+ entry: ./scripts/ci/pre_commit/check_airflow_imports.py
+ --pattern '^openlineage\.client\.(facet|run)'
+ --message "You should import from `airflow.providers.common.compat.openlineage.facet` instead."
additional_dependencies: ['rich>=12.4.4']
- id: check-airflow-providers-bug-report-template
name: Sort airflow-bug-report provider list
@@ -425,7 +432,9 @@ repos:
files: ^airflow-core/src/airflow/.*\.py$
require_serial: true
exclude: ^providers/.*/src/airflow/providers/
- entry: ./scripts/ci/pre_commit/check_cncf_k8s_used_for_k8s_executor_only.py
+ entry: ./scripts/ci/pre_commit/check_airflow_imports.py
+ --pattern '^airflow\.providers\.cncf\.kubernetes' \
+ --message "Only few k8s executors exceptions are allowed to use `airflow.providers.cncf.kubernetes`."
additional_dependencies: ['rich>=12.4.4']
- id: update-local-yml-file
name: Update mounts in the local yml file
@@ -696,7 +705,10 @@ repos:
name: Check providers import hook lineage code from compat
description: Make sure you import from airflow.provider.common.compat.lineage.hook instead of
airflow.lineage.hook.
- entry: ./scripts/ci/pre_commit/check_get_lineage_collector_providers.py
+ entry: ./scripts/ci/pre_commit/check_airflow_imports.py
+ --pattern '^airflow\.lineage\.hook'
+ --message "Only TYPE_CHECKING imports from `airflow.lineage.hook` are allowed in providers."
+ --only_top_level
files: ^providers/.*/src/airflow/providers/.*\.py$
exclude: ^providers/common/compat/src/airflow/providers/common/compat/.*\.py$
additional_dependencies: ['rich>=12.4.4']
diff --git a/scripts/ci/pre_commit/check_airflow_imports.py b/scripts/ci/pre_commit/check_airflow_imports.py
new file mode 100755
index 00000000000..6f32a447381
--- /dev/null
+++ b/scripts/ci/pre_commit/check_airflow_imports.py
@@ -0,0 +1,95 @@
+#!/usr/bin/env python
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+import re
+import argparse
+
+sys.path.insert(0, str(Path(__file__).parent.resolve())) # make sure common_precommit_utils is imported
+from common_precommit_utils import (
+ console,
+ get_imports_from_file
+)
+
+
+def parse_args():
+ parser = argparse.ArgumentParser(description="Check forbidden import patterns in Python files.")
+ parser.add_argument("files", nargs="+", type=Path, help="Python source files to check.")
+ parser.add_argument(
+ "--pattern",
+ required=True,
+ help="Regular expression pattern to match forbidden imports (e.g., '^airflow\\.kubernetes')."
+ )
+ parser.add_argument(
+ "--message",
+ required=True,
+ help="Error message to show when a forbidden import is found."
+ )
+ parser.add_argument(
+ "--only_top_level",
+ action="store_true",
+ help="If set, only top-level imports will be analyzed."
+ )
+ return parser.parse_args()
+
+
+def main() -> int:
+ args = parse_args()
+ errors: list[str] = []
+
+ try:
+ pattern = re.compile(args.pattern)
+ except re.error as e:
+ console.print(f"[red]Invalid regex pattern:[/] {e}")
+ return 2
+
+ for path in args.files:
+ import_count = 0
+ local_error_count = 0
+
+ try:
+ imports = get_imports_from_file(path, only_top_level=args.only_top_level)
+ except Exception as e:
+ console.print(f"[red]Failed to parse {path}: {e}[/]")
+ return 2
+
+ for imp in imports:
+ import_count += 1
+ if pattern.search(imp):
+ local_error_count += 1
+ errors.append(f"{path}: ({imp})")
+
+ console.print(f"[blue]{path}:[/] Import count: {import_count}, error_count {local_error_count}")
+
+ if errors:
+ console.print(f"[red]Some files contain forbidden imports matching pattern '{args.pattern}'.[/]")
+ console.print(args.message)
+ console.print("Error summary:")
+ for error in errors:
+ console.print(error)
+ return 1
+
+ console.print("[green]All good!")
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
\ No newline at end of file
diff --git a/scripts/ci/pre_commit/check_airflow_k8s_not_used.py b/scripts/ci/pre_commit/check_airflow_k8s_not_used.py
deleted file mode 100755
index e03c3009fdb..00000000000
--- a/scripts/ci/pre_commit/check_airflow_k8s_not_used.py
+++ /dev/null
@@ -1,81 +0,0 @@
-#!/usr/bin/env python
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-from __future__ import annotations
-
-import ast
-import sys
-from typing import NamedTuple
-
-from rich.console import Console
-
-console = Console(color_system="standard", width=200)
-
-
-class ImportTuple(NamedTuple):
- module: list[str]
- name: list[str]
- alias: str
-
-
-def get_imports(path: str):
- with open(path) as fh:
- root = ast.parse(fh.read(), path)
-
- for node in ast.iter_child_nodes(root):
- if isinstance(node, ast.Import):
- module: list[str] = node.names[0].name.split(".") if node.names else []
- elif isinstance(node, ast.ImportFrom) and node.module:
- module = node.module.split(".")
- else:
- continue
-
- for n in node.names: # type: ignore[attr-defined]
- yield ImportTuple(module=module, name=n.name.split("."), alias=n.asname)
-
-
-errors: list[str] = []
-
-
-def main() -> int:
- for path in sys.argv[1:]:
- import_count = 0
- local_error_count = 0
- for imp in get_imports(path):
- import_count += 1
- if len(imp.module) > 1:
- if imp.module[:2] == ["airflow", "kubernetes"]:
- local_error_count += 1
- errors.append(f"{path}: ({'.'.join(imp.module)})")
- console.print(f"[blue]{path}:[/] Import count: {import_count}, error_count {local_error_count}")
- if errors:
- console.print(
- "[red]Some files imports from `airflow.kubernetes`.[/]\n"
- "You should only import kubernetes code from `airflow.providers.cncf.kubernetes`."
- )
- console.print("Error summary:")
- for error in errors:
- console.print(error)
- return 1
- else:
- console.print("[green]All good!")
- return 0
-
-
-if __name__ == "__main__":
- sys.exit(main())
diff --git a/scripts/ci/pre_commit/check_cncf_k8s_used_for_k8s_executor_only.py b/scripts/ci/pre_commit/check_cncf_k8s_used_for_k8s_executor_only.py
deleted file mode 100755
index a452c648cdf..00000000000
--- a/scripts/ci/pre_commit/check_cncf_k8s_used_for_k8s_executor_only.py
+++ /dev/null
@@ -1,81 +0,0 @@
-#!/usr/bin/env python
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-from __future__ import annotations
-
-import ast
-import sys
-from typing import NamedTuple
-
-from rich.console import Console
-
-console = Console(color_system="standard", width=200)
-
-
-class ImportTuple(NamedTuple):
- module: list[str]
- name: list[str]
- alias: str
-
-
-def get_imports(path: str):
- with open(path) as fh:
- root = ast.parse(fh.read(), path)
-
- for node in ast.iter_child_nodes(root):
- if isinstance(node, ast.Import):
- module: list[str] = node.names[0].name.split(".") if node.names else []
- elif isinstance(node, ast.ImportFrom) and node.module:
- module = node.module.split(".")
- else:
- continue
-
- for n in node.names: # type: ignore[attr-defined]
- yield ImportTuple(module=module, name=n.name.split("."), alias=n.asname)
-
-
-errors: list[str] = []
-
-
-def main() -> int:
- for path in sys.argv[1:]:
- import_count = 0
- local_error_count = 0
- for imp in get_imports(path):
- import_count += 1
- if len(imp.module) > 3:
- if imp.module[:4] == ["airflow", "providers", "cncf", "kubernetes"]:
- local_error_count += 1
- errors.append(f"{path}: ({'.'.join(imp.module)})")
- console.print(f"[blue]{path}:[/] Import count: {import_count}, error_count {local_error_count}")
- if errors:
- console.print(
- "[red]Some files imports from `airflow.providers.cncf.kubernetes` and they are not allowed.[/]\n"
- "Only few k8s executors exceptions are allowed to use `airflow.providers.cncf.kubernetes`."
- )
- console.print("Error summary:")
- for error in errors:
- console.print(error)
- return 1
- else:
- console.print("[green]All good!")
- return 0
-
-
-if __name__ == "__main__":
- sys.exit(main())
diff --git a/scripts/ci/pre_commit/check_common_compat_used_for_openlineage.py b/scripts/ci/pre_commit/check_common_compat_used_for_openlineage.py
deleted file mode 100755
index 127a2b11cd3..00000000000
--- a/scripts/ci/pre_commit/check_common_compat_used_for_openlineage.py
+++ /dev/null
@@ -1,88 +0,0 @@
-#!/usr/bin/env python
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-from __future__ import annotations
-
-import ast
-import sys
-from typing import NamedTuple
-
-from rich.console import Console
-
-console = Console(color_system="standard", width=200)
-
-
-class ImportTuple(NamedTuple):
- module: list[str]
- name: list[str]
- alias: str
-
-
-def get_imports(path: str):
- with open(path) as fh:
- root = ast.parse(fh.read(), path)
-
- for node in ast.iter_child_nodes(root):
- if isinstance(node, ast.Import):
- module: list[str] = node.names[0].name.split(".") if node.names else []
- elif isinstance(node, ast.ImportFrom) and node.module:
- module = node.module.split(".")
- else:
- continue
-
- for n in node.names: # type: ignore[attr-defined]
- yield ImportTuple(module=module, name=n.name.split("."), alias=n.asname)
-
-
-errors: list[str] = []
-
-EXCEPTIONS = ["airflow/providers/common/compat/openlineage/facet.py"]
-
-
-def main() -> int:
- for path in sys.argv[1:]:
- import_count = 0
- local_error_count = 0
- for imp in get_imports(path):
- import_count += 1
- if len(imp.module) > 2:
- if imp.module[:3] == ["openlineage", "client", "facet"] or imp.module[:3] == [
- "openlineage",
- "client",
- "run",
- ]:
- if path not in EXCEPTIONS:
- local_error_count += 1
- errors.append(f"{path}: ({'.'.join(imp.module)})")
- console.print(f"[blue]{path}:[/] Import count: {import_count}, error_count {local_error_count}")
- if errors:
- console.print(
- "[red]Some files imports from `openlineage.client.facet` or `openlineage.client.run`. which are deprecated.[/]\n"
- "You should import from `airflow.providers.common.compat.openlineage.facet` instead."
- )
- console.print("Error summary:")
- for error in errors:
- console.print(error)
- return 1
- else:
- console.print("[green]All good!")
- return 0
-
-
-if __name__ == "__main__":
- sys.exit(main())
diff --git a/scripts/ci/pre_commit/check_get_lineage_collector_providers.py b/scripts/ci/pre_commit/check_get_lineage_collector_providers.py
deleted file mode 100755
index b7c811f6f79..00000000000
--- a/scripts/ci/pre_commit/check_get_lineage_collector_providers.py
+++ /dev/null
@@ -1,83 +0,0 @@
-#!/usr/bin/env python
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-from __future__ import annotations
-
-import ast
-import sys
-from pathlib import Path
-from typing import NamedTuple
-
-sys.path.insert(0, str(Path(__file__).parent.resolve()))
-from common_precommit_utils import console, initialize_breeze_precommit
-
-initialize_breeze_precommit(__name__, __file__)
-
-
-class ImportTuple(NamedTuple):
- module: list[str]
- name: list[str]
- alias: str
-
-
-def get_toplevel_imports(path: str):
- with open(path) as fh:
- root = ast.parse(fh.read(), path)
-
- for node in ast.iter_child_nodes(root):
- if isinstance(node, ast.Import):
- module: list[str] = node.names[0].name.split(".") if node.names else []
- elif isinstance(node, ast.ImportFrom) and node.module:
- module = node.module.split(".")
- else:
- continue
-
- for n in node.names: # type: ignore[attr-defined]
- yield ImportTuple(module=module, name=n.name.split("."), alias=n.asname)
-
-
-errors: list[str] = []
-
-
-def main() -> int:
- for path in sys.argv[1:]:
- import_count = 0
- local_error_count = 0
- for imp in get_toplevel_imports(path):
- import_count += 1
- if len(imp.module) > 2:
- if imp.module[:3] == ["airflow", "lineage", "hook"]:
- local_error_count += 1
- errors.append(f"{path}: ({'.'.join(imp.module)})")
- console.print(f"[blue]{path}:[/] Import count: {import_count}, error_count {local_error_count}")
- if errors:
- console.print(
- "[red]Some providers files import directly top level from `airflow.lineage.hook` and they are not allowed.[/]\n"
- "Only TYPE_CHECKING imports from `airflow.lineage.hook` is allowed in providers."
- )
- console.print("Error summary:")
- for error in errors:
- console.print(error)
- return 1
- else:
- console.print("[green]All good!")
- return 0
-
-
-if __name__ == "__main__":
- sys.exit(main())
diff --git a/scripts/ci/pre_commit/common_precommit_utils.py b/scripts/ci/pre_commit/common_precommit_utils.py
index cec8fdc0239..69e5e425e1b 100644
--- a/scripts/ci/pre_commit/common_precommit_utils.py
+++ b/scripts/ci/pre_commit/common_precommit_utils.py
@@ -300,3 +300,42 @@ def get_all_provider_info_dicts() -> dict[str, dict]:
if provider_info["state"] != "suspended":
providers[provider_id] = provider_info
return providers
+
+
+def get_imports_from_file(file_path: Path, *, only_top_level: bool) -> list[str]:
+ """
+ Returns list of all imports in file.
+
+ For following code:
+ import os
+ from collections import defaultdict
+ import numpy as np
+ from pandas import DataFrame as DF
+
+ def inner():
+ import json
+ from pathlib import Path, PurePath
+ from __future__ import annotations
+
+ When only_top_level = False then returns
+ ['os', 'collections.defaultdict', 'numpy', 'pandas.DataFrame']
+ When only_top_level = False then returns
+ ['os', 'collections.defaultdict', 'numpy', 'pandas.DataFrame', 'json', 'pathlib.Path', 'pathlib.PurePath']
+ """
+ root = ast.parse(file_path.read_text(), file_path.name)
+ imports: list[str] = []
+
+ nodes = ast.iter_child_nodes(root) if only_top_level else ast.walk(root)
+ for node in nodes:
+ if isinstance(node, ast.Import):
+ for alias in node.names:
+ imports.append(alias.name)
+ elif isinstance(node, ast.ImportFrom):
+ if node.module == "__future__":
+ continue
+ for alias in node.names:
+ name = alias.name
+ fullname = f"{node.module}.{name}" if node.module else name
+ imports.append(fullname)
+
+ return imports
\ No newline at end of file
diff --git a/scripts/ci/pre_commit/update_providers_dependencies.py b/scripts/ci/pre_commit/update_providers_dependencies.py
index 9c1e5d05adb..13de8d07016 100755
--- a/scripts/ci/pre_commit/update_providers_dependencies.py
+++ b/scripts/ci/pre_commit/update_providers_dependencies.py
@@ -28,7 +28,7 @@ from __future__ import annotations
import json
import os
import sys
-from ast import Import, ImportFrom, NodeVisitor, parse
+import ast
from collections import defaultdict
from pathlib import Path
from typing import Any
@@ -41,6 +41,7 @@ from common_precommit_utils import (
AIRFLOW_PROVIDERS_ROOT_PATH,
AIRFLOW_ROOT_PATH,
console,
+ get_imports_from_file
)
AIRFLOW_PROVIDERS_IMPORT_PREFIX = "airflow.providers."
@@ -67,38 +68,6 @@ ALL_PROVIDERS: dict[str, dict[str, Any]] = defaultdict(lambda: defaultdict())
ALL_PROVIDER_FILES: list[Path] = []
-class ImportFinder(NodeVisitor):
- """
- AST visitor that collects all imported names in its imports
- """
-
- def __init__(self) -> None:
- self.imports: list[str] = []
- self.handled_import_exception = list[str]
- self.tried_imports: list[str] = []
-
- def process_import(self, import_name: str) -> None:
- self.imports.append(import_name)
-
- def get_import_name_from_import_from(self, node: ImportFrom) -> list[str]:
- import_names: list[str] = []
- for alias in node.names:
- name = alias.name
- fullname = f"{node.module}.{name}" if node.module else name
- import_names.append(fullname)
- return import_names
-
- def visit_Import(self, node: Import):
- for alias in node.names:
- self.process_import(alias.name)
-
- def visit_ImportFrom(self, node: ImportFrom):
- if node.module == "__future__":
- return
- for fullname in self.get_import_name_from_import_from(node):
- self.process_import(fullname)
-
-
def load_pyproject_toml(pyproject_toml_file_path: Path) -> dict[str, Any]:
try:
import tomllib
@@ -164,13 +133,6 @@ def get_provider_id_from_import(import_name: str, file_path: Path) -> str | None
return provider_id
-def get_imports_from_file(file_path: Path) -> list[str]:
- root = parse(file_path.read_text(), file_path.name)
- visitor = ImportFinder()
- visitor.visit(root)
- return visitor.imports
-
-
def get_provider_id_from_path(file_path: Path) -> str | None:
"""
Get the provider id from the path of the file it belongs to.