You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2025/07/01 04:58:06 UTC
(airflow) branch main updated: Add more dependency reports (#52606)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 448a1843bad Add more dependency reports (#52606)
448a1843bad is described below
commit 448a1843bada35c792136ef497fd12d1ba9524d5
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Tue Jul 1 06:57:59 2025 +0200
Add more dependency reports (#52606)
The dependency reports of ours will be different for:
* different python versions
* different constraint modes
We change the job to produce the reports into matrix of jobs
producing reports for all combinations of those.
---
.github/workflows/finalize-tests.yml | 16 ++++--
dev/constraints-updated-version-check.py | 85 +++++++++++++++++++-------------
2 files changed, 64 insertions(+), 37 deletions(-)
diff --git a/.github/workflows/finalize-tests.yml b/.github/workflows/finalize-tests.yml
index bdfd0bbe79e..bab0d82bd7a 100644
--- a/.github/workflows/finalize-tests.yml
+++ b/.github/workflows/finalize-tests.yml
@@ -136,7 +136,11 @@ jobs:
runs-on: ${{ fromJSON(inputs.runners) }}
needs: [update-constraints]
if: inputs.upgrade-to-newer-dependencies == 'true' && inputs.platform == 'linux/amd64'
- name: "Dependency upgrade summary"
+ name: "Dependencies ${{ matrix.python-version }}:${{ matrix.constraints-mode }}"
+ strategy:
+ matrix:
+ python-version: ${{ fromJson(inputs.python-versions) }}
+ constraints-mode: ["constraints", "constraints-source-providers", "constraints-no-providers"]
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
steps:
@@ -151,14 +155,18 @@ jobs:
uses: ./.github/actions/prepare_breeze_and_image
with:
platform: ${{ inputs.platform }}
- python: ${{ inputs.default-python-version }}
+ python: ${{ matrix.python-version }}
use-uv: ${{ inputs.use-uv }}
- name: "Dependency upgrade summary"
shell: bash
run: >
- breeze run uv run /opt/airflow/dev/constraints-updated-version-check.py --explain-why
+ breeze run uv run /opt/airflow/dev/constraints-updated-version-check.py
+ --python-version "${MATRIX_PYTHON_VERSION}"
+ --airflow-constraints-mode "${MATRIX_CONSTRAINTS_MODE}" --explain-why
env:
- PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
+ MATRIX_PYTHON_VERSION: "${{ matrix.python-version }}"
+ MATRIX_CONSTRAINTS_MODE: "${{ matrix.constraints-mode }}"
+ VERBOSE: "false"
push-buildx-cache-to-github-registry:
name: Push Regular Image Cache ${{ inputs.platform }}
diff --git a/dev/constraints-updated-version-check.py b/dev/constraints-updated-version-check.py
index bf68c9b067a..35630854b13 100755
--- a/dev/constraints-updated-version-check.py
+++ b/dev/constraints-updated-version-check.py
@@ -56,16 +56,20 @@ def parse_constraints_generation_date(lines):
return None
-def get_constraints_file(python_version):
- url = f"https://raw.githubusercontent.com/apache/airflow/refs/heads/constraints-main/constraints-{python_version}.txt"
+def get_constraints_file(python_version, airflow_constraints_mode="constraints"):
+ url = f"https://raw.githubusercontent.com/apache/airflow/refs/heads/constraints-main/{airflow_constraints_mode}-{python_version}.txt"
try:
response = urllib.request.urlopen(url)
return response.read().decode("utf-8").splitlines()
except HTTPError as e:
if e.code == 404:
- console.print(f"[bold red]Error: Constraints file for Python {python_version} not found.[/]")
+ console.print(
+ f"[bold red]Error: Constraints file for Python {python_version} and mode {airflow_constraints_mode} not found.[/]"
+ )
console.print(f"[bold red]URL: {url}[/]")
- console.print("[bold red]Please check if the Python version is correct and the file exists.[/]")
+ console.print(
+ "[bold red]Please check if the Python version and constraints mode are correct and the file exists.[/]"
+ )
else:
console.print(f"[bold red]HTTP Error: {e.code} - {e.reason}[/]")
exit(1)
@@ -163,13 +167,16 @@ def get_first_newer_release_date_str(releases, current_version):
def main(
python_version: str,
+ airflow_constraints_mode: str,
mode: str,
selected_packages: set[str] | None = None,
explain_why: bool = False,
verbose: bool = False,
):
- lines = get_constraints_file(python_version)
+ lines = get_constraints_file(python_version, airflow_constraints_mode)
constraints_date = parse_constraints_generation_date(lines)
+ console.print(f"[bold cyan]Python version:[/] [white]{python_version}[/]")
+ console.print(f"[bold cyan]Constraints mode:[/] [white]{airflow_constraints_mode}[/]\n")
if constraints_date:
console.print(
f"[bold cyan]Constraints file generation date:[/] [white]{constraints_date.strftime('%Y-%m-%d %H:%M:%S')}[/]"
@@ -181,7 +188,7 @@ def main(
print_table_header(format_str, headers, total_width)
outdated_count, skipped_count, explanations = process_packages(
- packages, constraints_date, mode, explain_why, verbose, col_widths, format_str
+ packages, constraints_date, mode, explain_why, verbose, col_widths, format_str, python_version
)
print_table_footer(total_width, len(packages), outdated_count, skipped_count, mode)
@@ -267,6 +274,7 @@ def process_packages(
verbose: bool,
col_widths: dict,
format_str: str,
+ python_version: str,
) -> tuple[int, int, list[str]]:
import subprocess
import tempfile
@@ -319,6 +327,10 @@ def process_packages(
if not dep_added:
new_lines.append(f' "{pkg}=={latest_version}",')
pyproject_path.write_text("\n".join(new_lines) + "\n")
+ if verbose:
+ console.print(
+ f"[cyan]Fixed {pkg} at {latest_version} in [white]{pyproject_path}[/] [dim](pyproject.toml)[/]"
+ )
airflow_pyproject = Path(__file__).parent.parent / "pyproject.toml"
airflow_pyproject = airflow_pyproject.resolve()
@@ -369,6 +381,7 @@ def process_packages(
run_uv_sync,
update_pyproject_dependency,
verbose,
+ python_version,
)
explanations.append(explanation)
except HTTPError as e:
@@ -430,6 +443,7 @@ def explain_package_upgrade(
run_uv_sync,
update_pyproject_dependency,
verbose: bool,
+ python_version: str,
) -> str:
explanation = (
f"[bold blue]\n--- Explaining for {pkg} (current: {pinned_version}, latest: {latest_version}) ---[/]"
@@ -452,6 +466,8 @@ def explain_package_upgrade(
"--resolution",
"highest",
"--refresh",
+ "--python",
+ python_version,
],
cwd=repo_root,
)
@@ -470,13 +486,15 @@ def explain_package_upgrade(
"--resolution",
"highest",
"--refresh",
+ "--python",
+ python_version,
],
cwd=repo_root,
)
(temp_dir_path / "uv_sync_after.txt").write_text(after_result.stdout + after_result.stderr)
if after_result.returncode == 0:
explanation += f"\n[bold yellow]Package {pkg} can be upgraded from {pinned_version} to {latest_version} without conflicts.[/]."
- else:
+ if after_result.returncode != 0 or verbose:
explanation += f"\n[yellow]uv sync output for {pkg}=={latest_version}:[/]\n"
explanation += after_result.stdout + after_result.stderr
return explanation
@@ -485,46 +503,47 @@ def explain_package_upgrade(
@click.command()
@click.option(
"--python-version",
- required=False,
+ type=click.Choice(["3.10", "3.11", "3.12", "3.13"]),
default="3.10",
- help="Python version to check constraints for (e.g., 3.12)",
+ help="Python version to check constraints for (e.g. 3.10, 3.11, 3.12, 3.13).",
+)
+@click.option(
+ "--airflow-constraints-mode",
+ type=click.Choice(
+ ["constraints", "constraints-no-providers", "constraints-source-providers"], case_sensitive=False
+ ),
+ default="constraints",
+ show_default=True,
+ help="Constraints mode to use: constraints, constraints-no-providers, constraints-source-providers.",
)
@click.option(
"--mode",
- type=click.Choice(["full", "diff-constraints", "diff-all"]),
- default="diff-constraints",
+ type=click.Choice(["full", "diff-all", "diff-constraints"], case_sensitive=False),
+ default="full",
show_default=True,
- help="Operation mode: full, diff-constraints, or diff-all.",
+ help="Report mode: full, diff-all, diff-constraints.",
)
@click.option(
- "--selected-packages",
- required=False,
- default=None,
- help="Comma-separated list of package names to check (e.g., 'requests,flask'). If not set, all packages are checked.",
+ "--package",
+ multiple=True,
+ help="Only check specific package(s). Can be used multiple times.",
)
@click.option(
- "--explain-why",
- is_flag=True,
+ "--explain-why/--no-explain-why",
default=False,
- help="For each selected package, attempts to upgrade to the latest version and explains why it cannot be upgraded.",
+ help="Show explanations for outdated packages.",
)
@click.option(
- "--verbose",
- is_flag=True,
+ "--verbose/--no-verbose",
default=False,
- help="Print the temporary pyproject.toml file used for each package when running --explain-why.",
+ help="Show verbose output.",
)
-def cli(
- python_version: str,
- mode: str,
- selected_packages: str | None,
- explain_why: bool,
- verbose: bool,
-):
- selected_packages_set = None
- if selected_packages:
- selected_packages_set = set(pkg.strip() for pkg in selected_packages.split(",") if pkg.strip())
- main(python_version, mode, selected_packages_set, explain_why, verbose)
+def cli(python_version, airflow_constraints_mode, mode, package, explain_why, verbose):
+ if os.environ.get("CI", "false") == "true":
+ # Show output outside the group in CI
+ print("::endgroup::")
+ selected_packages = set(package) if package else None
+ main(python_version, airflow_constraints_mode, mode, selected_packages, explain_why, verbose)
if __name__ == "__main__":