You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2026/04/28 17:54:34 UTC
(airflow) branch v3-2-test updated: [v3-2-test] Make extract_versions.py work for explicit --version backfills (#65989) (#66007)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v3-2-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/v3-2-test by this push:
new af286977ff6 [v3-2-test] Make extract_versions.py work for explicit --version backfills (#65989) (#66007)
af286977ff6 is described below
commit af286977ff6696fb5fb7fadc7f59ff4d75728ba4
Author: github-actions[bot] <41...@users.noreply.github.com>
AuthorDate: Tue Apr 28 19:54:26 2026 +0200
[v3-2-test] Make extract_versions.py work for explicit --version backfills (#65989) (#66007)
(cherry picked from commit 743ec1f9cd677fab92af9036ccdc0a12ef42502c)
Co-authored-by: Kaxil Naik <ka...@gmail.com>
---
dev/registry/extract_versions.py | 45 ++++++++++++++++++++++---
dev/registry/tests/test_extract_versions.py | 51 +++++++++++++++++++++++++++++
2 files changed, 91 insertions(+), 5 deletions(-)
diff --git a/dev/registry/extract_versions.py b/dev/registry/extract_versions.py
index 5fb5719a95f..216a6a9575e 100644
--- a/dev/registry/extract_versions.py
+++ b/dev/registry/extract_versions.py
@@ -58,11 +58,21 @@ except ImportError:
from extract_metadata import fetch_provider_inventory, read_connection_urls, resolve_connection_docs_url
from registry_tools.types import MODULE_LEVEL_SECTIONS, TYPE_SUFFIXES
+SCRIPT_DIR = Path(__file__).parent
AIRFLOW_ROOT = Path(__file__).parent.parent.parent
PROVIDERS_DIR = AIRFLOW_ROOT / "providers"
REGISTRY_DIR = AIRFLOW_ROOT / "registry"
OUTPUT_DIR = REGISTRY_DIR / "src" / "_data" / "versions"
+# Same candidate order as extract_parameters.py and extract_metadata.py: the
+# `Download existing providers.json` workflow step writes to dev/registry/
+# (= SCRIPT_DIR), so prefer that. Fall back to the eleventy data dir when
+# running locally after a full extract pass has populated it.
+PROVIDERS_JSON_CANDIDATES = [
+ SCRIPT_DIR / "providers.json",
+ REGISTRY_DIR / "src" / "_data" / "providers.json",
+]
+
def build_provider_id_to_path_map() -> dict[str, str]:
"""Scan providers/ for provider.yaml to build provider_id -> directory_path mapping."""
@@ -434,10 +444,16 @@ def main():
parser.add_argument("--all-versions", action="store_true", help="Extract all versions")
args = parser.parse_args()
- # Load providers.json
- providers_path = REGISTRY_DIR / "src" / "_data" / "providers.json"
- if not providers_path.exists():
- print(f"ERROR: {providers_path} not found. Run extract_metadata.py first.")
+ # Load providers.json from the first candidate path that exists.
+ providers_path = next((p for p in PROVIDERS_JSON_CANDIDATES if p.exists()), None)
+ if providers_path is None:
+ candidates = "\n ".join(str(p) for p in PROVIDERS_JSON_CANDIDATES)
+ print(
+ "ERROR: providers.json not found in any of:\n "
+ f"{candidates}\n"
+ "Run extract_metadata.py first, or download from the registry S3 bucket "
+ "to dev/registry/providers.json."
+ )
sys.exit(1)
with open(providers_path) as f:
@@ -479,8 +495,21 @@ def main():
elif args.version == latest_version:
print(f" {args.version} is the latest version (already in providers.json), skipping")
continue
+ elif git_tag_exists(f"providers-{pid}/{args.version}"):
+ # The cached providers.json predates this version (typical
+ # backfill scenario: the version was released after the last
+ # full registry build, so it's missing from `versions:` in
+ # the cached snapshot). Trust the git tag and proceed.
+ print(
+ f" {args.version} not in cached providers.json {all_versions} "
+ f"but providers-{pid}/{args.version} tag exists; extracting."
+ )
+ versions_to_extract = [args.version]
else:
- print(f" {args.version} not found in {pid} versions: {all_versions}")
+ print(
+ f" {args.version} not found in {pid} versions: {all_versions} "
+ f"and providers-{pid}/{args.version} tag does not exist"
+ )
total_skipped += 1
continue
elif args.all_versions:
@@ -497,6 +526,12 @@ def main():
if not extraction_tasks:
print(f"\nDone: {total_extracted} versions extracted, {total_skipped} skipped")
+ # Explicit `--version` with nothing extracted is a hard failure -- the
+ # caller asked for a specific version that we couldn't produce. Failing
+ # loud here surfaces the issue at extract time rather than later when
+ # `aws s3 sync` errors on a missing source path.
+ if args.version and total_skipped > 0:
+ sys.exit(1)
return
max_workers = min(8, len(extraction_tasks))
diff --git a/dev/registry/tests/test_extract_versions.py b/dev/registry/tests/test_extract_versions.py
new file mode 100644
index 00000000000..45a2e26a2ea
--- /dev/null
+++ b/dev/registry/tests/test_extract_versions.py
@@ -0,0 +1,51 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Unit tests for dev/registry/extract_versions.py."""
+
+from __future__ import annotations
+
+from extract_versions import (
+ AIRFLOW_ROOT,
+ PROVIDERS_JSON_CANDIDATES,
+ SCRIPT_DIR,
+)
+
+
+class TestProvidersJsonCandidates:
+ """Lock down the lookup order for providers.json.
+
+ Regression test for the workflow path mismatch where the
+ `Download existing providers.json` step writes to dev/registry/ but
+ the script previously only checked the eleventy data dir.
+ """
+
+ def test_dev_registry_path_first(self):
+ # The workflow's S3 download writes here; checked first so CI runs
+ # work without having to do a full local extract pass first.
+ assert PROVIDERS_JSON_CANDIDATES[0] == SCRIPT_DIR / "providers.json"
+
+ def test_eleventy_data_dir_fallback(self):
+ # Local dev convenience: after a full extract pass, the data dir
+ # has providers.json and we don't need to also keep a copy in
+ # dev/registry/.
+ assert PROVIDERS_JSON_CANDIDATES[1] == AIRFLOW_ROOT / "registry" / "src" / "_data" / "providers.json"
+
+ def test_no_other_candidates(self):
+ # Adding silently to the list could mask path mismatches that
+ # should be caught and fixed at the source. Match siblings (extract_
+ # parameters.py, extract_metadata.py) which use exactly these two.
+ assert len(PROVIDERS_JSON_CANDIDATES) == 2