(airflow) 01/02: Validate is it possible to build wheel from sdist distribution

[ta...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

taragolis pushed a commit to branch validate-sdist-to-wheel-2
in repository https://gitbox.apache.org/repos/asf/airflow.git

commit 7986fe800fb327fbd823b3201858f064f4632782
Author: Andrey Anshin <An...@taragol.is>
AuthorDate: Thu Feb 15 16:53:25 2024 +0400

    Validate is it possible to build wheel from sdist distribution
---
 .../commands/release_management_commands.py        | 103 ++++++++++++++++++++-
 1 file changed, 98 insertions(+), 5 deletions(-)

diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
index 3edeed633a..04669e3113 100644
--- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
@@ -23,14 +23,16 @@ import random
 import re
 import shutil
 import sys
+import tempfile
 import textwrap
 import time
+import venv
 from collections import defaultdict
 from copy import deepcopy
 from datetime import datetime
 from pathlib import Path
 from subprocess import DEVNULL
-from typing import IO, TYPE_CHECKING, Any, Generator, Iterable, NamedTuple
+from typing import IO, TYPE_CHECKING, Any, Generator, Iterable, Iterator, Literal, NamedTuple
 
 import click
 from rich.progress import Progress
@@ -288,6 +290,34 @@ AIRFLOW_BUILD_DOCKERFILE_PATH = AIRFLOW_SOURCES_ROOT / "airflow-build-dockerfile
 AIRFLOW_BUILD_DOCKERFILE_IGNORE_PATH = AIRFLOW_SOURCES_ROOT / "airflow-build-dockerfile.dockerignore"
 
 
+class DistributionPackageInfo(NamedTuple):
+    filepath: Path
+    package: str
+    version: Version
+    dist_type: Literal["sdist", "wheel"]
+
+    @classmethod
+    def from_sdist(cls, filepath: Path) -> DistributionPackageInfo:
+        from packaging.utils import parse_sdist_filename
+
+        package, version = parse_sdist_filename(filepath.name)
+        return cls(
+            filepath=filepath.resolve().absolute(), package=package, version=version, dist_type="sdist"
+        )
+
+    @classmethod
+    def from_wheel(cls, filepath: Path) -> DistributionPackageInfo:
+        from packaging.utils import parse_wheel_filename
+
+        package, version, *_ = parse_wheel_filename(filepath.name)
+        return cls(
+            filepath=filepath.resolve().absolute(), package=package, version=version, dist_type="wheel"
+        )
+
+    def __str__(self):
+        return f"{self.filepath.name}: {self.dist_type} {self.package} {self.version}"
+
+
 def _build_local_build_image():
     # This is security feature.
     #
@@ -374,6 +404,58 @@ def _build_airflow_packages_with_hatch(
     )
 
 
+def _dist_packages(
+    *, package_format: str, build_type: Literal["airflow", "providers"]
+) -> Iterator[DistributionPackageInfo]:
+    if build_type == "airflow":
+        default_glob_pattern = "apache[_-]airflow-[0-9]"
+    else:
+        default_glob_pattern = "apache[_-]airflow[_-]providers"
+
+    if package_format in ["sdist", "both"]:
+        for file in DIST_DIR.glob(f"{default_glob_pattern}*tar.gz"):
+            if not file.is_file():
+                continue
+            yield DistributionPackageInfo.from_sdist(filepath=file)
+    if package_format in ["wheel", "both"]:
+        for file in DIST_DIR.glob(f"{default_glob_pattern}*whl"):
+            if not file.is_file():
+                continue
+            yield DistributionPackageInfo.from_wheel(filepath=file)
+
+
+def _check_sdist_to_wheel(dist_info: DistributionPackageInfo):
+    if dist_info.dist_type != "sdist":
+        return
+
+    with tempfile.TemporaryDirectory() as tmp_dir_name:
+        venv_path = Path(tmp_dir_name) / ".venv"
+        venv.EnvBuilder(with_pip=True).create(venv_path)
+        python_path = venv_path / "bin" / "python"
+        if not python_path.exists():
+            msg = f"Python interpreter is not exist in path {python_path}"
+            raise FileNotFoundError(msg)
+        pip_command = (python_path.__fspath__(), "-m", "pip")
+        run_command([*pip_command, "install", f"pip=={AIRFLOW_PIP_VERSION}", "-U"], check=True)
+        run_command(
+            [
+                *pip_command,
+                "wheel",
+                "--wheel-dir",
+                str(tmp_dir_name),
+                "--no-deps",
+                "--no-cache",
+                "--no-binary",
+                dist_info.package,
+                dist_info.filepath.__fspath__(),
+            ],
+            check=True,
+            # We should run `pip wheel` outside of Project directory for avoid the case
+            # when some files presented into the project directory, but not included in sdist.
+            cwd=str(tmp_dir_name),
+        )
+
+
 @release_management.command(
     name="prepare-airflow-package",
     help="Prepare sdist/whl package of Airflow.",
@@ -406,8 +488,19 @@ def prepare_airflow_packages(
             version_suffix_for_pypi=version_suffix_for_pypi,
         )
     get_console().print("[success]Successfully prepared Airflow packages:")
-    for file in sorted(DIST_DIR.glob("apache_airflow*")):
-        get_console().print(file.name)
+    packages = tuple(_dist_packages(package_format=package_format, build_type="airflow"))
+    for dist_info in packages:
+        get_console().print(str(dist_info))
+    get_console().print()
+    if package_format == "wheel":
+        return
+
+    for dist_info in packages:
+        if dist_info.dist_type == "sdist":
+            get_console().print(
+                f"[info]Validate build wheel from sdist distribution: {dist_info.filepath.name!r}[/]"
+            )
+            _check_sdist_to_wheel(dist_info)
     get_console().print()
 
 
@@ -711,8 +804,8 @@ def prepare_provider_packages(
         sys.exit(0)
     get_console().print("\n[success]Successfully built packages!\n\n")
     get_console().print("\n[info]Packages available in dist:\n")
-    for file in sorted(DIST_DIR.glob("apache*")):
-        get_console().print(file.name)
+    for dist_info in _dist_packages(package_format=package_format, build_type="providers"):
+        get_console().print(str(dist_info))
     get_console().print()