You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2026/08/09 08:22:35 UTC
(airflow-site-archive) branch replace-dependabot-with-update-workflow created (now bfc203e397a)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a change to branch replace-dependabot-with-update-workflow
in repository https://gitbox.apache.org/repos/asf/airflow-site-archive.git
at bfc203e397a Replace Dependabot with a sparse-checkout update workflow
This branch includes the following new commits:
new bfc203e397a Replace Dependabot with a sparse-checkout update workflow
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(airflow-site-archive) 01/01: Replace Dependabot with a sparse-checkout update workflow
Posted by po...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch replace-dependabot-with-update-workflow
in repository https://gitbox.apache.org/repos/asf/airflow-site-archive.git
commit bfc203e397ae1a3763b6bcefe4bb706bb178efdf
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Sun Aug 9 16:13:26 2026 +0800
Replace Dependabot with a sparse-checkout update workflow
Dependabot clones the whole repository before its file fetcher runs. At
826k files and 47 GB that never finishes inside a Dependabot job, so the
runs time out and the pinned actions and pre-commit hooks fall behind -
only one Dependabot PR has landed here since April.
There is no knob in dependabot.yml that avoids the clone, so this drops
the config and does the same job from a workflow that only ever needs the
configuration files:
* .github/workflows/update-pinned-deps.yml runs weekly (and on demand),
once per target branch, on a sparse blobless checkout of .github.
* scripts/update_pinned_deps.py resolves each SHA-pinned action and
pre-commit hook to the newest release outside a 7-day cooldown window -
the same cooldown the Dependabot config used - and rewrites the pins in
place, preserving the `# vX.Y.Z` / `# frozen: vX.Y.Z` style. Branch
pins such as `# main` move along their branch instead.
Pull requests opened with GITHUB_TOKEN do not trigger other workflows, so
the workflow runs pre-commit on the changed files itself before opening
the PR.
Dependabot security alerts are unaffected - they do not come from
dependabot.yml.
Co-Authored-By: Claude Opus 5 (1M context) <no...@anthropic.com>
---
.github/dependabot.yml | 63 -------
.github/workflows/update-pinned-deps.yml | 131 ++++++++++++++
scripts/update_pinned_deps.py | 300 +++++++++++++++++++++++++++++++
3 files changed, 431 insertions(+), 63 deletions(-)
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
deleted file mode 100644
index e2da1f1f0fc..00000000000
--- a/.github/dependabot.yml
+++ /dev/null
@@ -1,63 +0,0 @@
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
----
-version: 2
-updates:
- - package-ecosystem: github-actions
- cooldown:
- default-days: 7
- directory: /
- schedule:
- interval: daily
- groups:
- github-actions:
- patterns:
- - ".github/*"
- - package-ecosystem: github-actions
- cooldown:
- default-days: 7
- directory: /
- target-branch: staging
- schedule:
- interval: daily
- groups:
- github-actions:
- patterns:
- - ".github/*"
- - package-ecosystem: pre-commit
- cooldown:
- default-days: 7
- directory: /
- schedule:
- interval: weekly
- groups:
- pre-commit-hooks:
- patterns:
- - ".pre-commit-config.yaml"
- - package-ecosystem: pre-commit
- cooldown:
- default-days: 7
- directory: /
- target-branch: staging
- schedule:
- interval: weekly
- groups:
- pre-commit-hooks:
- patterns:
- - ".pre-commit-config.yaml"
diff --git a/.github/workflows/update-pinned-deps.yml b/.github/workflows/update-pinned-deps.yml
new file mode 100644
index 00000000000..4351eb2a7ec
--- /dev/null
+++ b/.github/workflows/update-pinned-deps.yml
@@ -0,0 +1,131 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+---
+name: Update pinned dependencies
+# Replaces Dependabot for GitHub Actions and pre-commit hooks. Dependabot clones
+# the whole repository before its file fetcher runs, and at 800k+ files / 47+ GB
+# that never finishes inside a Dependabot job - so updates simply stopped
+# landing. This workflow only needs the configuration files, which a sparse
+# blobless checkout gets in seconds.
+on: # yamllint disable-line rule:truthy
+ schedule:
+ - cron: '0 6 * * 1' # Mondays, 06:00 UTC
+ workflow_dispatch:
+ inputs:
+ cooldown-days:
+ description: "Ignore releases published more recently than this many days"
+ default: "7"
+ type: string
+permissions:
+ contents: read
+concurrency:
+ group: update-pinned-deps
+ cancel-in-progress: false
+jobs:
+ update:
+ name: "Update pins on ${{ matrix.branch }}"
+ runs-on: ubuntu-latest
+ if: github.repository == 'apache/airflow-site-archive'
+ permissions:
+ contents: write # push the update branch
+ pull-requests: write # open the update pull request
+ strategy:
+ fail-fast: false
+ matrix:
+ branch: [main, staging]
+ env:
+ TARGET_BRANCH: ${{ matrix.branch }}
+ UPDATE_BRANCH: update-pinned-deps-${{ matrix.branch }}
+ COOLDOWN_DAYS: ${{ inputs.cooldown-days || '7' }}
+ steps:
+ - name: "Checkout ${{ matrix.branch }} configuration"
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ # Only the configuration is needed - never check out docs-archive here.
+ with:
+ ref: ${{ matrix.branch }}
+ fetch-depth: 1
+ filter: blob:none
+ sparse-checkout: |
+ .github
+ persist-credentials: false
+
+ - name: "Checkout ${{ github.sha }} for scripts"
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ # The updater always comes from the ref this workflow runs from, so it
+ # does not have to exist on the branch being updated yet.
+ with:
+ ref: ${{ github.sha }}
+ path: ./tooling
+ fetch-depth: 1
+ filter: blob:none
+ sparse-checkout: |
+ scripts
+ persist-credentials: false
+
+ - name: "Install uv"
+ run: curl -LsSf https://astral.sh/uv/install.sh | sh
+
+ - name: "Resolve the latest versions outside the cooldown window"
+ id: update
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ run: |
+ uv run --no-project ./tooling/scripts/update_pinned_deps.py \
+ --repo-root . \
+ --cooldown-days "${COOLDOWN_DAYS}" \
+ --summary-file "${RUNNER_TEMP}/summary.md"
+
+ - name: "Lint the updated configuration"
+ if: steps.update.outputs.updated == 'true'
+ run: |
+ uv tool install pre-commit --with pre-commit-uv
+ # Pull requests opened with GITHUB_TOKEN do not trigger other
+ # workflows, so the checks that would run on the pull request are run
+ # here instead - on the changed files only, because `--all-files`
+ # would walk all 800k+ files in the index.
+ git diff --name-only -z | xargs -0 pre-commit run --color always --files
+
+ - name: "Open the update pull request"
+ if: steps.update.outputs.updated == 'true'
+ env:
+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ run: |
+ set -euo pipefail
+ git config user.name "github-actions[bot]"
+ git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+ git switch -c "${UPDATE_BRANCH}"
+ git commit -a -m "Update pinned GitHub Actions and pre-commit hooks"
+ git push --force "https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" \
+ "HEAD:refs/heads/${UPDATE_BRANCH}"
+ if [[ -n "$(gh pr list --head "${UPDATE_BRANCH}" --base "${TARGET_BRANCH}" \
+ --state open --json number --jq '.[].number')" ]]; then
+ echo "Pull request already open - the force push updated it"
+ exit 0
+ fi
+ {
+ echo "Automated update of the SHA-pinned GitHub Actions and pre-commit hooks,"
+ echo "honouring a ${COOLDOWN_DAYS}-day cooldown on new releases."
+ echo
+ cat "${RUNNER_TEMP}/summary.md"
+ echo
+ echo "Opened by [\`update-pinned-deps.yml\`](.github/workflows/update-pinned-deps.yml)."
+ echo "Pre-commit ran on the changed files in that workflow run, because pull"
+ echo "requests opened with \`GITHUB_TOKEN\` do not trigger \`Basic checks\`."
+ } > "${RUNNER_TEMP}/pr-body.md"
+ gh pr create --base "${TARGET_BRANCH}" --head "${UPDATE_BRANCH}" \
+ --title "Update pinned GitHub Actions and pre-commit hooks (${TARGET_BRANCH})" \
+ --body-file "${RUNNER_TEMP}/pr-body.md"
diff --git a/scripts/update_pinned_deps.py b/scripts/update_pinned_deps.py
new file mode 100755
index 00000000000..7fd47a0819b
--- /dev/null
+++ b/scripts/update_pinned_deps.py
@@ -0,0 +1,300 @@
+#!/usr/bin/env python3
+# /// script
+# requires-python = ">=3.12"
+# dependencies = []
+# ///
+"""Update SHA-pinned GitHub Actions and pre-commit hooks to their latest versions.
+
+This replaces Dependabot for the `github-actions` and `pre-commit` ecosystems.
+Dependabot clones the whole repository before its file fetcher runs, and this
+repository is far too big for that (800k+ files, 47+ GB), so every Dependabot
+job times out and updates never land. This script only ever reads
+`.github/workflows`, `.github/actions` and `.pre-commit-config.yaml`, so it runs
+from a sparse checkout in seconds.
+
+Only pinned references are touched, and the pinning style is preserved:
+
+ uses: actions/checkout@<sha> # v7.0.1 -> newest release tag
+ uses: apache/some-action@<sha> # main -> newest commit on that branch
+ rev: <sha> # frozen: v1.29.0 -> newest release tag
+
+A cooldown window (7 days by default, matching the Dependabot config this
+replaces) keeps us off releases that are still hot.
+
+Set GITHUB_TOKEN (or GH_TOKEN) to avoid the unauthenticated API rate limit.
+"""
+
+from __future__ import annotations
+
+import argparse
+import http.client
+import json
+import os
+import re
+import sys
+import urllib.error
+import urllib.request
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+from time import sleep
+
+API_ROOT = "https://api.github.com"
+API_ATTEMPTS = 3
+# Enough history to find the newest release outside the cooldown window without
+# pulling megabytes of release notes for repositories that publish a lot.
+API_PAGE_SIZE = 30
+
+# `uses: owner/repo[/sub/path]@<40-hex sha> # <tag-or-branch>`
+USES_RE = re.compile(
+ r"(?P<prefix>uses:\s+)"
+ r"(?P<repo>[A-Za-z0-9._-]+/[A-Za-z0-9._-]+)"
+ r"(?P<subpath>(?:/[^@\s]+)?)"
+ r"@(?P<sha>[0-9a-f]{40})"
+ r"(?P<gap>\s+#\s+)"
+ r"(?P<comment>\S+)"
+)
+# `- repo: https://github.com/owner/repo`
+PRE_COMMIT_REPO_RE = re.compile(r"^\s*-\s+repo:\s+https://github\.com/(?P<repo>[^\s/]+/[^\s/]+?)(?:\.git)?\s*$")
+# `rev: <sha> # frozen: v1.29.0`
+PRE_COMMIT_REV_RE = re.compile(r"^(?P<prefix>\s*rev:\s+)(?P<rev>\S+)(?P<gap>\s+#\s+frozen:\s+)(?P<tag>\S+)\s*$")
+# Strict release versions only - no prereleases, no moving major/minor tags.
+VERSION_RE = re.compile(r"^v?(?P<version>\d+(?:\.\d+)*)$")
+
+
+class GitHubError(RuntimeError):
+ """Raised when the GitHub API cannot answer a question we need answered."""
+
+
+def api(path: str, **params: str | int) -> list | dict | None:
+ """Call the GitHub REST API, returning None for a 404.
+
+ Transient failures (dropped connections, truncated responses, 5xx) are
+ retried - a single flaky read should not fail the whole update run.
+ """
+ url = f"{API_ROOT}/{path}"
+ if params:
+ url += "?" + "&".join(f"{key}={value}" for key, value in params.items())
+ request = urllib.request.Request(url, headers={"Accept": "application/vnd.github+json"})
+ token = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN")
+ if token:
+ request.add_header("Authorization", f"Bearer {token}")
+ for attempt in range(1, API_ATTEMPTS + 1):
+ try:
+ with urllib.request.urlopen(request, timeout=60) as response:
+ return json.loads(response.read())
+ except urllib.error.HTTPError as error:
+ if error.code == 404:
+ return None
+ if error.code < 500 or attempt == API_ATTEMPTS:
+ raise GitHubError(f"GET {url} failed: {error.code} {error.reason}") from error
+ except (urllib.error.URLError, http.client.HTTPException, TimeoutError) as error:
+ if attempt == API_ATTEMPTS:
+ raise GitHubError(f"GET {url} failed: {error}") from error
+ sleep(attempt)
+ return None
+
+
+def parse_version(tag: str) -> tuple[int, ...] | None:
+ """Turn `v1.29.0` into `(1, 29, 0)`, or return None if it is not a release version."""
+ match = VERSION_RE.match(tag)
+ if not match:
+ return None
+ return tuple(int(part) for part in match.group("version").split("."))
+
+
+def released_at(entry: dict) -> datetime:
+ return datetime.fromisoformat(entry["published_at"])
+
+
+def latest_release_tag(repo: str, cutoff: datetime) -> str | None:
+ """Newest release tag of `repo` that was published before `cutoff`.
+
+ Releases are preferred over tags because they carry a publication date and
+ exclude drafts, prereleases and moving major tags such as `v7`. Repositories
+ that do not publish releases fall back to tags, dated by their commit.
+ """
+ releases = api(f"repos/{repo}/releases", per_page=API_PAGE_SIZE) or []
+ candidates = [
+ (version, release["tag_name"])
+ for release in releases
+ if not release["draft"] and not release["prerelease"] and released_at(release) <= cutoff
+ for version in [parse_version(release["tag_name"])]
+ if version
+ ]
+ if candidates:
+ return max(candidates)[1]
+
+ tags = api(f"repos/{repo}/tags", per_page=API_PAGE_SIZE) or []
+ dated = sorted(
+ ((version, tag["name"], tag["commit"]["sha"]) for tag in tags for version in [parse_version(tag["name"])] if version),
+ reverse=True,
+ )
+ for _, name, sha in dated:
+ commit = api(f"repos/{repo}/commits/{sha}")
+ if commit and datetime.fromisoformat(commit["commit"]["committer"]["date"]) <= cutoff:
+ return name
+ return None
+
+
+def tag_sha(repo: str, tag: str) -> str:
+ """Commit SHA a tag points at, dereferencing annotated tags."""
+ reference = api(f"repos/{repo}/git/ref/tags/{tag}")
+ if not reference:
+ raise GitHubError(f"tag {tag} not found in {repo}")
+ target = reference["object"]
+ if target["type"] == "tag":
+ target = api(f"repos/{repo}/git/tags/{target['sha']}")["object"]
+ return target["sha"]
+
+
+def latest_branch_sha(repo: str, branch: str, cutoff: datetime) -> str | None:
+ """Newest commit on `branch` that is older than `cutoff`."""
+ commits = api(f"repos/{repo}/commits", sha=branch, until=cutoff.strftime("%Y-%m-%dT%H:%M:%SZ"), per_page=1)
+ return commits[0]["sha"] if commits else None
+
+
+class Updater:
+ """Resolves and applies pin updates, collecting a report as it goes."""
+
+ def __init__(self, cutoff: datetime, dry_run: bool = False) -> None:
+ self.cutoff = cutoff
+ self.dry_run = dry_run
+ self.updates: list[tuple[str, str, str]] = []
+ self.resolved: dict[tuple[str, str], tuple[str, str] | None] = {}
+
+ def write(self, path: Path, content: str) -> None:
+ if not self.dry_run:
+ path.write_text(content)
+
+ def resolve(self, repo: str, current_tag: str, current_sha: str) -> tuple[str, str] | None:
+ """Return the (tag, sha) this pin should move to, or None to leave it alone."""
+ key = (repo, current_tag)
+ if key not in self.resolved:
+ self.resolved[key] = self._resolve(repo, current_tag, current_sha)
+ target = self.resolved[key]
+ return target if target and target[1] != current_sha else None
+
+ def _resolve(self, repo: str, current_tag: str, current_sha: str) -> tuple[str, str] | None:
+ current_version = parse_version(current_tag)
+ if current_version is None:
+ # A branch pin such as `# main` - move it along the branch instead.
+ sha = latest_branch_sha(repo, current_tag, self.cutoff)
+ return (current_tag, sha) if sha else None
+ tag = latest_release_tag(repo, self.cutoff)
+ if tag is None:
+ print(f" ! {repo}: no release older than the cooldown window, left at {current_tag}")
+ return None
+ if parse_version(tag) < current_version:
+ print(f" ! {repo}: newest eligible release {tag} is older than the pinned {current_tag}, left alone")
+ return None
+ return tag, tag_sha(repo, tag)
+
+ def record(self, name: str, before_tag: str, before_sha: str, after_tag: str, after_sha: str) -> None:
+ if before_tag == after_tag:
+ # A branch pin, or a tag that was moved to a different commit - the
+ # tag alone would read as a no-op, so show what actually changed.
+ before, after = f"{before_tag} ({before_sha[:7]})", f"{after_tag} ({after_sha[:7]})"
+ else:
+ before, after = before_tag, after_tag
+ # The same action is typically pinned in several places - report it once.
+ if (name, before, after) in self.updates:
+ return
+ self.updates.append((name, before, after))
+ print(f" * {name}: {before} -> {after}")
+
+ def update_workflows(self, paths: list[Path]) -> None:
+ for path in sorted(paths):
+ content = original = path.read_text()
+ for match in list(USES_RE.finditer(original)):
+ repo, current_tag, current_sha = match["repo"], match["comment"], match["sha"]
+ target = self.resolve(repo, current_tag, current_sha)
+ if not target:
+ continue
+ tag, sha = target
+ replacement = (
+ f"{match['prefix']}{repo}{match['subpath']}@{sha}{match['gap']}{tag}"
+ )
+ content = content.replace(match.group(0), replacement)
+ self.record(f"{repo}{match['subpath']}", current_tag, current_sha, tag, sha)
+ if content != original:
+ self.write(path, content)
+
+ def update_pre_commit(self, path: Path) -> None:
+ if not path.exists():
+ return
+ lines = path.read_text().splitlines(keepends=True)
+ repo: str | None = None
+ for index, line in enumerate(lines):
+ repo_match = PRE_COMMIT_REPO_RE.match(line)
+ if repo_match:
+ repo = repo_match["repo"]
+ continue
+ rev_match = PRE_COMMIT_REV_RE.match(line)
+ if not (repo and rev_match):
+ continue
+ target = self.resolve(repo, rev_match["tag"], rev_match["rev"])
+ if not target:
+ continue
+ tag, sha = target
+ lines[index] = f"{rev_match['prefix']}{sha}{rev_match['gap']}{tag}\n"
+ self.record(repo, rev_match["tag"], rev_match["rev"], tag, sha)
+ self.write(path, "".join(lines))
+
+ def report(self) -> str:
+ if not self.updates:
+ return "No updates available outside the cooldown window.\n"
+ widths = [max(len(row[column]) for row in self.updates) for column in range(3)]
+ header = f"| {'Dependency':<{widths[0]}} | {'From':<{widths[1]}} | {'To':<{widths[2]}} |\n"
+ divider = f"|{'-' * (widths[0] + 2)}|{'-' * (widths[1] + 2)}|{'-' * (widths[2] + 2)}|\n"
+ rows = "".join(
+ f"| {name:<{widths[0]}} | {before:<{widths[1]}} | {after:<{widths[2]}} |\n"
+ for name, before, after in self.updates
+ )
+ return header + divider + rows
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+ parser.add_argument("--cooldown-days", type=int, default=7, help="ignore releases newer than this many days")
+ parser.add_argument("--repo-root", type=Path, default=Path(__file__).parent.parent, help="repository root")
+ parser.add_argument("--summary-file", type=Path, help="write the markdown summary table here")
+ parser.add_argument("--dry-run", action="store_true", help="report what would change without writing files")
+ args = parser.parse_args()
+
+ now = datetime.now(UTC)
+ cutoff = now - timedelta(days=args.cooldown_days)
+ print(f"Updating pins to the newest versions released before {cutoff:%Y-%m-%d %H:%M} UTC "
+ f"({args.cooldown_days}-day cooldown)")
+
+ workflows = sorted(
+ {
+ *(args.repo_root / ".github" / "workflows").glob("*.y*ml"),
+ *(args.repo_root / ".github" / "actions").rglob("*.y*ml"),
+ }
+ )
+ pre_commit_config = args.repo_root / ".pre-commit-config.yaml"
+
+ updater = Updater(cutoff, dry_run=args.dry_run)
+ try:
+ print("GitHub Actions:")
+ updater.update_workflows(workflows)
+ print("Pre-commit hooks:")
+ updater.update_pre_commit(pre_commit_config)
+ except GitHubError as error:
+ print(f"error: {error}", file=sys.stderr)
+ return 1
+
+ summary = updater.report()
+ print(summary, end="")
+ if args.summary_file:
+ args.summary_file.write_text(summary)
+ if step_summary := os.environ.get("GITHUB_STEP_SUMMARY"):
+ Path(step_summary).write_text(f"### Pinned dependency updates\n\n{summary}")
+ if github_output := os.environ.get("GITHUB_OUTPUT"):
+ with Path(github_output).open("a") as output:
+ output.write(f"updated={'true' if updater.updates else 'false'}\n")
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())