You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2025/04/30 22:18:25 UTC
(airflow) branch v3-0-test updated: [v3-0-test] Avoid too many requests when refreshing constraint from GitHub (#49967) (#49977)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v3-0-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/v3-0-test by this push:
new 033e70e8c4f [v3-0-test] Avoid too many requests when refreshing constraint from GitHub (#49967) (#49977)
033e70e8c4f is described below
commit 033e70e8c4f273fd2318446d6279e5b476fa725c
Author: github-actions[bot] <41...@users.noreply.github.com>
AuthorDate: Thu May 1 00:18:19 2025 +0200
[v3-0-test] Avoid too many requests when refreshing constraint from GitHub (#49967) (#49977)
We started to get the 429 errors (too many requests) when refreshing
constraints. This change should increase the limits with authentication
(cherry picked from commit a377ffffdf924f4070d78b7ce7e733c4be8fbefb)
Co-authored-by: Jarek Potiuk <ja...@potiuk.com>
---
.github/workflows/basic-tests.yml | 2 ++
...ease-management_generate-providers-metadata.svg | 26 +++++++++++++---------
...ease-management_generate-providers-metadata.txt | 2 +-
.../commands/release_management_commands.py | 13 ++++++++---
.../commands/release_management_commands_config.py | 5 ++++-
dev/breeze/src/airflow_breeze/utils/github.py | 20 ++++++++++++++---
6 files changed, 49 insertions(+), 19 deletions(-)
diff --git a/.github/workflows/basic-tests.yml b/.github/workflows/basic-tests.yml
index 176921df494..7efc3deedc1 100644
--- a/.github/workflows/basic-tests.yml
+++ b/.github/workflows/basic-tests.yml
@@ -353,6 +353,8 @@ jobs:
git remote add apache https://github.com/apache/airflow.git
git fetch apache --tags
breeze release-management generate-providers-metadata --refresh-constraints
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: "Fetch all git tags for origin"
run: git fetch --tags >/dev/null 2>&1 || true
- name: "Test airflow core issue generation automatically"
diff --git a/dev/breeze/doc/images/output_release-management_generate-providers-metadata.svg b/dev/breeze/doc/images/output_release-management_generate-providers-metadata.svg
index 9c6a8105daa..4f1f26d3456 100644
--- a/dev/breeze/doc/images/output_release-management_generate-providers-metadata.svg
+++ b/dev/breeze/doc/images/output_release-management_generate-providers-metadata.svg
@@ -1,4 +1,4 @@
-<svg class="rich-terminal" viewBox="0 0 1482 416.0" xmlns="http://www.w3.org/2000/svg">
+<svg class="rich-terminal" viewBox="0 0 1482 440.4" xmlns="http://www.w3.org/2000/svg">
<!-- Generated with Rich https://www.textualize.io -->
<style>
@@ -43,7 +43,7 @@
<defs>
<clipPath id="breeze-release-management-generate-providers-metadata-clip-terminal">
- <rect x="0" y="0" width="1463.0" height="365.0" />
+ <rect x="0" y="0" width="1463.0" height="389.4" />
</clipPath>
<clipPath id="breeze-release-management-generate-providers-metadata-line-0">
<rect x="0" y="1.5" width="1464" height="24.65"/>
@@ -87,9 +87,12 @@
<clipPath id="breeze-release-management-generate-providers-metadata-line-13">
<rect x="0" y="318.7" width="1464" height="24.65"/>
</clipPath>
+<clipPath id="breeze-release-management-generate-providers-metadata-line-14">
+ <rect x="0" y="343.1" width="1464" height="24.65"/>
+ </clipPath>
</defs>
- <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1" x="1" y="1" width="1480" height="414" rx="8"/><text class="breeze-release-management-generate-providers-metadata-title" fill="#c5c8c6" text-anchor="middle" x="740" y="27">Command: release-management generate-providers-metadata</text>
+ <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1" x="1" y="1" width="1480" height="438.4" rx="8"/><text class="breeze-release-management-generate-providers-metadata-title" fill="#c5c8c6" text-anchor="middle" x="740" y="27">Command: release-management generate-providers-metadata</text>
<g transform="translate(26,22)">
<circle cx="0" cy="0" r="7" fill="#ff5f57"/>
<circle cx="22" cy="0" r="7" fill="#febc2e"/>
@@ -106,14 +109,15 @@
</text><text class="breeze-release-management-generate-providers-metadata-r1" x="1464" y="117.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-providers-metadata-line-4)">
</text><text class="breeze-release-management-generate-providers-metadata-r5" x="0" y="142" textLength="24.4" clip-path="url(#breeze-release-management-generate-providers-metadata-line-5)">╭─</text><text class="breeze-release-management-generate-providers-metadata-r5" x="24.4" y="142" textLength="427" clip-path="url(#breeze-release-management-generate-providers-metadata-line-5)"> Generate providers metadata flags </text><text class="breeze-release-management-gene [...]
</text><text class="breeze-release-management-generate-providers-metadata-r5" x="0" y="166.4" textLength="12.2" clip-path="url(#breeze-release-management-generate-providers-metadata-line-6)">│</text><text class="breeze-release-management-generate-providers-metadata-r4" x="24.4" y="166.4" textLength="256.2" clip-path="url(#breeze-release-management-generate-providers-metadata-line-6)">--refresh-constraints</text><text class="breeze-release-management-generate-providers-metadata-r1" x="329 [...]
-</text><text class="breeze-release-management-generate-providers-metadata-r5" x="0" y="190.8" textLength="12.2" clip-path="url(#breeze-release-management-generate-providers-metadata-line-7)">│</text><text class="breeze-release-management-generate-providers-metadata-r4" x="24.4" y="190.8" textLength="97.6" clip-path="url(#breeze-release-management-generate-providers-metadata-line-7)">--python</text><text class="breeze-release-management-generate-providers-metadata-r1" x="329.4" y="190.8" [...]
-</text><text class="breeze-release-management-generate-providers-metadata-r5" x="0" y="215.2" textLength="12.2" clip-path="url(#breeze-release-management-generate-providers-metadata-line-8)">│</text><text class="breeze-release-management-generate-providers-metadata-r6" x="329.4" y="215.2" textLength="976" clip-path="url(#breeze-release-management-generate-providers-metadata-line-8)">(3.6 | 3.7 | 3.8 | 3.9 | 3.10 | 3.11 | 3.12)&# [...]
-</text><text class="breeze-release-management-generate-providers-metadata-r5" x="0" y="239.6" textLength="1464" clip-path="url(#breeze-release-management-generate-providers-metadata-line-9)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-release-management-generate-providers-metadata-r1" x="1464" y="239.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-providers-metad [...]
-</text><text class="breeze-release-management-generate-providers-metadata-r5" x="0" y="264" textLength="24.4" clip-path="url(#breeze-release-management-generate-providers-metadata-line-10)">╭─</text><text class="breeze-release-management-generate-providers-metadata-r5" x="24.4" y="264" textLength="195.2" clip-path="url(#breeze-release-management-generate-providers-metadata-line-10)"> Common options </text><text class="breeze-release-management-generate-providers-metadata-r [...]
-</text><text class="breeze-release-management-generate-providers-metadata-r5" x="0" y="288.4" textLength="12.2" clip-path="url(#breeze-release-management-generate-providers-metadata-line-11)">│</text><text class="breeze-release-management-generate-providers-metadata-r4" x="24.4" y="288.4" textLength="109.8" clip-path="url(#breeze-release-management-generate-providers-metadata-line-11)">--dry-run</text><text class="breeze-release-management-generate-providers-metadata-r7" x="158.6" y="288 [...]
-</text><text class="breeze-release-management-generate-providers-metadata-r5" x="0" y="312.8" textLength="12.2" clip-path="url(#breeze-release-management-generate-providers-metadata-line-12)">│</text><text class="breeze-release-management-generate-providers-metadata-r4" x="24.4" y="312.8" textLength="109.8" clip-path="url(#breeze-release-management-generate-providers-metadata-line-12)">--verbose</text><text class="breeze-release-management-generate-providers-metadata-r7" x="158.6" y="312 [...]
-</text><text class="breeze-release-management-generate-providers-metadata-r5" x="0" y="337.2" textLength="12.2" clip-path="url(#breeze-release-management-generate-providers-metadata-line-13)">│</text><text class="breeze-release-management-generate-providers-metadata-r4" x="24.4" y="337.2" textLength="73.2" clip-path="url(#breeze-release-management-generate-providers-metadata-line-13)">--help</text><text class="breeze-release-management-generate-providers-metadata-r7" x="158.6" y="337.2" [...]
-</text><text class="breeze-release-management-generate-providers-metadata-r5" x="0" y="361.6" textLength="1464" clip-path="url(#breeze-release-management-generate-providers-metadata-line-14)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-release-management-generate-providers-metadata-r1" x="1464" y="361.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-providers-meta [...]
+</text><text class="breeze-release-management-generate-providers-metadata-r5" x="0" y="190.8" textLength="12.2" clip-path="url(#breeze-release-management-generate-providers-metadata-line-7)">│</text><text class="breeze-release-management-generate-providers-metadata-r4" x="24.4" y="190.8" textLength="170.8" clip-path="url(#breeze-release-management-generate-providers-metadata-line-7)">--github-token</text><text class="breeze-release-management-generate-providers-metadata-r1" x="329.4" y=" [...]
+</text><text class="breeze-release-management-generate-providers-metadata-r5" x="0" y="215.2" textLength="12.2" clip-path="url(#breeze-release-management-generate-providers-metadata-line-8)">│</text><text class="breeze-release-management-generate-providers-metadata-r4" x="24.4" y="215.2" textLength="97.6" clip-path="url(#breeze-release-management-generate-providers-metadata-line-8)">--python</text><text class="breeze-release-management-generate-providers-metadata-r1" x="329.4" y="215.2" [...]
+</text><text class="breeze-release-management-generate-providers-metadata-r5" x="0" y="239.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-providers-metadata-line-9)">│</text><text class="breeze-release-management-generate-providers-metadata-r6" x="329.4" y="239.6" textLength="976" clip-path="url(#breeze-release-management-generate-providers-metadata-line-9)">(3.6 | 3.7 | 3.8 | 3.9 | 3.10 | 3.11 | 3.12)&# [...]
+</text><text class="breeze-release-management-generate-providers-metadata-r5" x="0" y="264" textLength="1464" clip-path="url(#breeze-release-management-generate-providers-metadata-line-10)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-release-management-generate-providers-metadata-r1" x="1464" y="264" textLength="12.2" clip-path="url(#breeze-release-management-generate-providers-metadata [...]
+</text><text class="breeze-release-management-generate-providers-metadata-r5" x="0" y="288.4" textLength="24.4" clip-path="url(#breeze-release-management-generate-providers-metadata-line-11)">╭─</text><text class="breeze-release-management-generate-providers-metadata-r5" x="24.4" y="288.4" textLength="195.2" clip-path="url(#breeze-release-management-generate-providers-metadata-line-11)"> Common options </text><text class="breeze-release-management-generate-providers-metada [...]
+</text><text class="breeze-release-management-generate-providers-metadata-r5" x="0" y="312.8" textLength="12.2" clip-path="url(#breeze-release-management-generate-providers-metadata-line-12)">│</text><text class="breeze-release-management-generate-providers-metadata-r4" x="24.4" y="312.8" textLength="109.8" clip-path="url(#breeze-release-management-generate-providers-metadata-line-12)">--dry-run</text><text class="breeze-release-management-generate-providers-metadata-r7" x="158.6" y="312 [...]
+</text><text class="breeze-release-management-generate-providers-metadata-r5" x="0" y="337.2" textLength="12.2" clip-path="url(#breeze-release-management-generate-providers-metadata-line-13)">│</text><text class="breeze-release-management-generate-providers-metadata-r4" x="24.4" y="337.2" textLength="109.8" clip-path="url(#breeze-release-management-generate-providers-metadata-line-13)">--verbose</text><text class="breeze-release-management-generate-providers-metadata-r7" x="158.6" y="337 [...]
+</text><text class="breeze-release-management-generate-providers-metadata-r5" x="0" y="361.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-providers-metadata-line-14)">│</text><text class="breeze-release-management-generate-providers-metadata-r4" x="24.4" y="361.6" textLength="73.2" clip-path="url(#breeze-release-management-generate-providers-metadata-line-14)">--help</text><text class="breeze-release-management-generate-providers-metadata-r7" x="158.6" y="361.6" [...]
+</text><text class="breeze-release-management-generate-providers-metadata-r5" x="0" y="386" textLength="1464" clip-path="url(#breeze-release-management-generate-providers-metadata-line-15)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-release-management-generate-providers-metadata-r1" x="1464" y="386" textLength="12.2" clip-path="url(#breeze-release-management-generate-providers-metadata [...]
</text>
</g>
</g>
diff --git a/dev/breeze/doc/images/output_release-management_generate-providers-metadata.txt b/dev/breeze/doc/images/output_release-management_generate-providers-metadata.txt
index 8d686a6b365..e885738de07 100644
--- a/dev/breeze/doc/images/output_release-management_generate-providers-metadata.txt
+++ b/dev/breeze/doc/images/output_release-management_generate-providers-metadata.txt
@@ -1 +1 @@
-572b36905a967caed880209b60618ede
+d4630907f51e9f17d442492905a3c47d
diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
index 96957d40df2..d89c089286e 100644
--- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
@@ -53,6 +53,7 @@ from airflow_breeze.commands.common_options import (
option_debug_resources,
option_dry_run,
option_github_repository,
+ option_github_token,
option_historical_python_version,
option_include_not_ready_providers,
option_include_removed_providers,
@@ -2657,7 +2658,9 @@ def generate_issue_content_core(
def get_all_constraint_files(
- refresh_constraints: bool, python_version: str
+ refresh_constraints: bool,
+ python_version: str,
+ github_token: str | None = None,
) -> tuple[list[str], dict[str, str]]:
if refresh_constraints:
shutil.rmtree(CONSTRAINTS_CACHE_PATH, ignore_errors=True)
@@ -2669,6 +2672,7 @@ def get_all_constraint_files(
if not download_constraints_file(
airflow_version=airflow_version,
python_version=python_version,
+ github_token=github_token,
include_provider_dependencies=True,
output_file=CONSTRAINTS_CACHE_PATH
/ f"constraints-{airflow_version}-python-{python_version}.txt",
@@ -2703,15 +2707,18 @@ def load_constraints(python_version: str) -> dict[str, dict[str, str]]:
is_flag=True,
help="Refresh constraints before generating metadata",
)
+@option_github_token
@option_historical_python_version
@option_dry_run
@option_verbose
-def generate_providers_metadata(refresh_constraints: bool, python: str | None):
+def generate_providers_metadata(refresh_constraints: bool, github_token: str | None, python: str | None):
metadata_dict: dict[str, dict[str, dict[str, str]]] = {}
if python is None:
python = DEFAULT_PYTHON_MAJOR_MINOR_VERSION
all_airflow_releases, airflow_release_dates = get_all_constraint_files(
- refresh_constraints=refresh_constraints, python_version=python
+ refresh_constraints=refresh_constraints,
+ python_version=python,
+ github_token=github_token,
)
constraints = load_constraints(python_version=python)
diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py
index a641322940e..c820901f063 100644
--- a/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py
+++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py
@@ -376,7 +376,10 @@ RELEASE_MANAGEMENT_PARAMETERS: dict[str, list[dict[str, str | list[str]]]] = {
}
],
"breeze release-management generate-providers-metadata": [
- {"name": "Generate providers metadata flags", "options": ["--refresh-constraints", "--python"]}
+ {
+ "name": "Generate providers metadata flags",
+ "options": ["--refresh-constraints", "--github-token", "--python"],
+ }
],
"breeze release-management start-rc-process": [
{
diff --git a/dev/breeze/src/airflow_breeze/utils/github.py b/dev/breeze/src/airflow_breeze/utils/github.py
index 516af2fdced..445ceecd2d6 100644
--- a/dev/breeze/src/airflow_breeze/utils/github.py
+++ b/dev/breeze/src/airflow_breeze/utils/github.py
@@ -40,13 +40,16 @@ def get_ga_output(name: str, value: Any) -> str:
return f"{output_name}={printed_value}"
-def download_file_from_github(tag: str, path: str, output_file: Path, timeout: int = 60) -> bool:
+def download_file_from_github(
+ tag: str, path: str, output_file: Path, github_token: str | None = None, timeout: int = 60
+) -> bool:
"""
Downloads a file from GitHub repository of Apache Airflow
:param tag: tag to download from
:param path: path of the file relative to the repository root
:param output_file: Path where the file should be downloaded
+ :param github_token: GitHub token to use for authentication
:param timeout: timeout in seconds for the download request, default is 60 seconds
:return: whether the file was successfully downloaded (False if the file is missing or error occurred)
"""
@@ -56,7 +59,12 @@ def download_file_from_github(tag: str, path: str, output_file: Path, timeout: i
get_console().print(f"[info]Downloading {url} to {output_file}")
if not get_dry_run():
try:
- response = requests.get(url, timeout=timeout)
+ # add github token to the request if provided
+ headers = {}
+ if github_token:
+ headers["Authorization"] = f"Bearer {github_token}"
+ headers["X-GitHub-Api-Version"] = "2022-11-28"
+ response = requests.get(url, headers=headers, timeout=timeout)
if response.status_code == 403:
get_console().print(
f"[error]The {url} is not accessible.This may be caused by either of:\n"
@@ -139,13 +147,18 @@ def get_active_airflow_versions(confirm: bool = True) -> tuple[list[str], dict[s
def download_constraints_file(
- airflow_version: str, python_version: str, include_provider_dependencies: bool, output_file: Path
+ airflow_version: str,
+ python_version: str,
+ github_token: str | None,
+ include_provider_dependencies: bool,
+ output_file: Path,
) -> bool:
"""
Downloads constraints file from GitHub repository of Apache Airflow
:param airflow_version: airflow version
:param python_version: python version
+ :param github_token: GitHub token
:param include_provider_dependencies: whether to include provider dependencies
:param output_file: the file where to store the constraint file
:return: true if the file was successfully downloaded
@@ -158,6 +171,7 @@ def download_constraints_file(
return download_file_from_github(
tag=constraints_tag,
path=constraints_file_path,
+ github_token=github_token,
output_file=output_file,
)