(airflow) 01/42: Move check provider yaml file to breeze shell (#35879)

[po...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

potiuk pushed a commit to branch v2-8-test
in repository https://gitbox.apache.org/repos/asf/airflow.git

commit 8bf64109ac0f839c8c0c39f00f9424d85d1eabca
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Mon Nov 27 16:16:27 2023 +0100

    Move check provider yaml file to breeze shell (#35879)
    
    (cherry picked from commit 7f8ba9deed62a66f07deb22e52b19628edd111f9)
---
 .pre-commit-config.yaml                            |  2 +-
 dev/breeze/tests/test_shell_params.py              | 16 ++++++
 .../pre_commit_check_provider_yaml_files.py        | 63 ++++++----------------
 3 files changed, 32 insertions(+), 49 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 805cc72ebd..53d655ae12 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -1063,7 +1063,7 @@ repos:
         entry: ./scripts/ci/pre_commit/pre_commit_check_provider_yaml_files.py
         language: python
         files: ^airflow/providers/.*/provider\.yaml$
-        additional_dependencies: ['rich>=12.4.4', 'inputimeout', 'pyyaml', 'jsonschema', 'filelock', 'markdown-it-py']
+        additional_dependencies: ['rich>=12.4.4']
         require_serial: true
       - id: update-migration-references
         name: Update migration ref doc
diff --git a/dev/breeze/tests/test_shell_params.py b/dev/breeze/tests/test_shell_params.py
index 2ce10ba66a..987884f4c8 100644
--- a/dev/breeze/tests/test_shell_params.py
+++ b/dev/breeze/tests/test_shell_params.py
@@ -170,6 +170,22 @@ console = Console(width=400, color_system="standard")
             },
             id="ENABLED_SYSTEMS empty by default even if they are None in ShellParams",
         ),
+        pytest.param(
+            {},
+            {},
+            {
+                "PYTHONWARNINGS": None,
+            },
+            id="PYTHONWARNINGS should not be set by default",
+        ),
+        pytest.param(
+            {"PYTHONWARNINGS": "default"},
+            {},
+            {
+                "PYTHONWARNINGS": "default",
+            },
+            id="PYTHONWARNINGS should be set when specified in environment",
+        ),
     ],
 )
 def test_shell_params_to_env_var_conversion(
diff --git a/scripts/ci/pre_commit/pre_commit_check_provider_yaml_files.py b/scripts/ci/pre_commit/pre_commit_check_provider_yaml_files.py
index c5126237d3..7c934c60fb 100755
--- a/scripts/ci/pre_commit/pre_commit_check_provider_yaml_files.py
+++ b/scripts/ci/pre_commit/pre_commit_check_provider_yaml_files.py
@@ -17,57 +17,24 @@
 # under the License.
 from __future__ import annotations
 
-import os
-import shlex
 import sys
 from pathlib import Path
 
-if __name__ not in ("__main__", "__mp_main__"):
-    raise SystemExit(
-        "This file is intended to be executed as an executable program. You cannot use it as a module."
-        f"To run this script, run the ./{__file__} command"
-    )
-
-AIRFLOW_SOURCES = Path(__file__).parents[3].resolve()
-GITHUB_REPOSITORY = os.environ.get("GITHUB_REPOSITORY", "apache/airflow")
-os.environ["SKIP_GROUP_OUTPUT"] = "true"
-
-if __name__ == "__main__":
-    sys.path.insert(0, str(AIRFLOW_SOURCES / "dev" / "breeze" / "src"))
-    from airflow_breeze.global_constants import DEFAULT_PYTHON_MAJOR_MINOR_VERSION, MOUNT_SELECTED
-    from airflow_breeze.params.shell_params import ShellParams
-    from airflow_breeze.utils.console import get_console
-    from airflow_breeze.utils.docker_command_utils import get_extra_docker_flags
-    from airflow_breeze.utils.run_utils import get_ci_image_for_pre_commits, run_command
+sys.path.insert(0, str(Path(__file__).parent.resolve()))
+from common_precommit_utils import console, initialize_breeze_precommit, run_command_via_breeze_shell
 
-    shell_params = ShellParams(python=DEFAULT_PYTHON_MAJOR_MINOR_VERSION)
+initialize_breeze_precommit(__name__, __file__)
 
-    cmd = "python3 /opt/airflow/scripts/in_container/run_provider_yaml_files_check.py"
-    if len(sys.argv) > 1:
-        cmd = cmd + " " + " ".join([shlex.quote(f) for f in sys.argv[1:]])
-    airflow_image = get_ci_image_for_pre_commits()
-    cmd_result = run_command(
-        [
-            "docker",
-            "run",
-            "-t",
-            *get_extra_docker_flags(mount_sources=MOUNT_SELECTED),
-            "-e",
-            "SKIP_ENVIRONMENT_INITIALIZATION=true",
-            "-e",
-            "PYTHONWARNINGS=default",
-            "--pull",
-            "never",
-            airflow_image,
-            "-c",
-            cmd,
-        ],
-        check=False,
-        env=shell_params.env_variables_for_docker_commands,
+files_to_test = sys.argv[1:]
+cmd_result = run_command_via_breeze_shell(
+    ["python3", "/opt/airflow/scripts/in_container/run_provider_yaml_files_check.py", *files_to_test],
+    backend="sqlite",
+    warn_image_upgrade_needed=True,
+    extra_env={"PYTHONWARNINGS": "default"},
+)
+if cmd_result.returncode != 0:
+    console.print(
+        "[warning]\nIf you see strange stacktraces above, "
+        "run `breeze ci-image build --python 3.8` and try again."
     )
-    if cmd_result.returncode != 0:
-        get_console().print(
-            "[warning]If you see strange stacktraces above, "
-            "run `breeze ci-image build --python 3.8` and try again."
-        )
-    sys.exit(cmd_result.returncode)
+sys.exit(cmd_result.returncode)