You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tooling.apache.org by ar...@apache.org on 2026/06/30 12:53:14 UTC
(tooling-trusted-releases) 02/03: #478 - allow releases of archived projects to be downloaded, and make sure they come from archive.a.o.
This is an automated email from the ASF dual-hosted git repository.
alitheg pushed a commit to branch arm
in repository https://gitbox.apache.org/repos/asf/tooling-trusted-releases.git
commit 629ca8a118c70233a9ffd05b71e0daab6494c440
Author: Alastair McFarlane <ar...@apache.org>
AuthorDate: Tue Jun 30 13:51:55 2026 +0100
#478 - allow releases of archived projects to be downloaded, and make sure they come from archive.a.o.
---
atr/constants.py | 1 +
atr/get/catalog.py | 3 +--
atr/paths.py | 6 ++++++
atr/shared/catalog.py | 21 +++++++++++++--------
atr/svn/dist.py | 6 ++++--
atr/util.py | 5 ++++-
tests/unit/test_catalog.py | 9 +++++----
tests/unit/test_decompose.py | 9 +++++++++
tests/unit/test_svn_publish_helpers.py | 17 +++++++++++++++++
9 files changed, 60 insertions(+), 17 deletions(-)
diff --git a/atr/constants.py b/atr/constants.py
index 869cf0e9..29231ea9 100644
--- a/atr/constants.py
+++ b/atr/constants.py
@@ -19,6 +19,7 @@ from typing import Final
CLOSER_LUA_URL: Final[str] = "https://www.apache.org/dyn/closer.lua"
DOWNLOADS_APACHE_URL: Final[str] = "https://downloads.apache.org"
+ARCHIVE_APACHE_URL: Final[str] = "https://archive.apache.org/dist"
SVN_DIST_PUBLIC_URL: Final[str] = "https://dist.apache.org/repos/dist/atr"
SVN_DIST_ROOT_URL: Final[str] = "https://dist.apache.org/repos/dist"
diff --git a/atr/get/catalog.py b/atr/get/catalog.py
index d3e12776..45516c52 100644
--- a/atr/get/catalog.py
+++ b/atr/get/catalog.py
@@ -24,7 +24,6 @@ import atr.api as api
import atr.blueprints.get as get
import atr.db as db
import atr.models.safe as safe
-import atr.models.sql as sql
import atr.shared.catalog as catalog
import atr.template as template
import atr.util as util
@@ -38,7 +37,7 @@ async def project(_session: web.Public, _catalog: Literal["catalog"], project_ke
The release catalogue for a project: versions and their artifacts.
"""
async with db.session() as data:
- project_obj = await data.project(key=str(project_key), status=sql.ProjectStatus.ACTIVE).demand(
+ project_obj = await data.project(key=str(project_key)).demand(
base.ASFQuartException(f"Project {project_key} not found", errorcode=404)
)
artifacts = await data.artifact(project_key=project_obj.key, _release=True).all()
diff --git a/atr/paths.py b/atr/paths.py
index 7e4ac812..e36478d2 100644
--- a/atr/paths.py
+++ b/atr/paths.py
@@ -65,6 +65,12 @@ def committee_downloads_url(host: str, committee: sql.Committee) -> str:
return f"https://{host}/downloads/{committee.key}"
+def archive_download_url(relpath: safe.RelPath) -> str:
+ # archive.apache.org keeps every release ever published, including those pruned from the
+ # live mirror, so an archived release's files are served from here regardless of host.
+ return f"{constants.ARCHIVE_APACHE_URL}/{relpath}"
+
+
def downloads_url(relpath: safe.RelPath) -> str:
# downloads.apache.org itself, not a mirror. Signatures, checksums and KEYS
# must come from here rather than a mirror, otherwise a bad mirror could serve
diff --git a/atr/shared/catalog.py b/atr/shared/catalog.py
index 203cda68..0d2aa29f 100644
--- a/atr/shared/catalog.py
+++ b/atr/shared/catalog.py
@@ -66,7 +66,9 @@ def assemble(
return CatalogProject(versions=versions, cycles=cycles, grouped=grouped)
-def _artifact(row: sql.Artifact, committee: sql.Committee | None, downloadable: bool) -> models.api.CatalogArtifact:
+def _artifact(
+ row: sql.Artifact, committee: sql.Committee | None, downloadable: bool, archived: bool
+) -> models.api.CatalogArtifact:
suffix = safe.RelPath(row.download_path_suffix) if row.download_path_suffix else None
artifact_url: str | None = None
signature_url: str | None = None
@@ -74,18 +76,20 @@ def _artifact(row: sql.Artifact, committee: sql.Committee | None, downloadable:
sbom_url: str | None = None
if downloadable and (committee is not None):
artifact_url = util.public_download_url(
- committee, suffix, util.DownloadFile.ARTIFACT, filename=row.artifact_path
+ committee, suffix, util.DownloadFile.ARTIFACT, filename=row.artifact_path, archived=archived
)
if row.signature_path:
signature_url = util.public_download_url(
- committee, suffix, util.DownloadFile.METADATA, filename=row.signature_path
+ committee, suffix, util.DownloadFile.METADATA, filename=row.signature_path, archived=archived
)
if row.checksum_path:
checksum_url = util.public_download_url(
- committee, suffix, util.DownloadFile.METADATA, filename=row.checksum_path
+ committee, suffix, util.DownloadFile.METADATA, filename=row.checksum_path, archived=archived
)
if row.sbom_path:
- sbom_url = util.public_download_url(committee, suffix, util.DownloadFile.METADATA, filename=row.sbom_path)
+ sbom_url = util.public_download_url(
+ committee, suffix, util.DownloadFile.METADATA, filename=row.sbom_path, archived=archived
+ )
return models.api.CatalogArtifact(
artifact_path=row.artifact_path,
classification=row.classification,
@@ -168,8 +172,9 @@ def _versions(
for version, rows in by_version.items():
release = next((row.release for row in rows if row.release is not None), None)
status = _status(release)
- # Archived versions aren't served from the download route, so show metadata only.
- downloadable = status == "released"
+ # Released files come off the live download route, archived ones off archive.apache.org;
+ # either way they're downloadable
+ downloadable = status in ("released", "archived")
cycle = cycles_by_key.get(release.cycle_key) if (release is not None) else None
svn_revisions = [row.svn_revision for row in rows if row.svn_revision is not None]
# CLE only resolves for a released-phase record, so historical svn-only
@@ -186,7 +191,7 @@ def _versions(
cle_url=cle_release_url(atr_host, release.project_key, str(version))
if (atr_host is not None) and cle_eligible and (release is not None)
else None,
- artifacts=[_artifact(row, committee, downloadable) for row in rows],
+ artifacts=[_artifact(row, committee, downloadable, status == "archived") for row in rows],
)
)
diff --git a/atr/svn/dist.py b/atr/svn/dist.py
index 73e6816a..9fc8eeac 100644
--- a/atr/svn/dist.py
+++ b/atr/svn/dist.py
@@ -82,10 +82,12 @@ class Decomposed:
def candidate_keys(committee: str, subproject: str | None) -> list[str]:
# Dist layout and projects.json keys don't always line up (commons/lang -> commons-lang, but
- # accumulo/accumulo-access is already its key), so try the obvious shapes; apache- is stripped
+ # accumulo/accumulo-access is already its key), so try the obvious shapes. apache-/apache_ come
+ # off and underscores become hyphens, so a dist name like apache_airflow_providers_alibaba lines
+ # up with the registry's airflow-providers-alibaba instead of seeding an underscore duplicate
if not subproject:
return [committee]
- bare = subproject.removeprefix("apache-")
+ bare = subproject.removeprefix("apache-").removeprefix("apache_").replace("_", "-")
keys: list[str] = []
for name in dict.fromkeys([bare, subproject]):
keys += [name, f"{committee}-{name}"]
diff --git a/atr/util.py b/atr/util.py
index 5c646541..ceaf3d3a 100644
--- a/atr/util.py
+++ b/atr/util.py
@@ -1029,9 +1029,12 @@ def public_download_url(
kind: DownloadFile,
filename: str | None = None,
host: str | None = None,
+ archived: bool = False,
) -> str:
# The single place that resolves where a release file is fetched from, by
- # lifecycle (beta vs released) and role (artifact vs metadata).
+ # lifecycle (archived vs beta vs released) and role (artifact vs metadata).
+ if archived:
+ return paths.archive_download_url(paths.committee_dist_relpath(committee, suffix, filename))
if not config.get().SVN_PUBLISH_URL:
# No SVN publishing configured (dev), so ATR serves the downloads itself.
url = paths.committee_downloads_url(host or config.get().APP_HOST, committee)
diff --git a/tests/unit/test_catalog.py b/tests/unit/test_catalog.py
index 24db804b..2f064412 100644
--- a/tests/unit/test_catalog.py
+++ b/tests/unit/test_catalog.py
@@ -88,7 +88,7 @@ def test_lifecycle_badge_prefers_lts_then_eol_then_active() -> None:
assert catalog._lifecycle_badge(_cycle("5.0"), _NOW) == "Active"
-def test_only_released_versions_are_downloadable() -> None:
+def test_released_and_archived_versions_are_downloadable() -> None:
project = _project()
released = _release(project, "5.0.2", cycle_key="cassandra-default", released=_NOW)
archived = _release(project, "4.1.7", cycle_key="cassandra-default", released=_NOW, archived=_NOW, is_archived=True)
@@ -99,8 +99,9 @@ def test_only_released_versions_are_downloadable() -> None:
versions = {str(v.version): v for v in catalog._versions(artifacts, {}, project.committee_key)}
+ # Released off the live route, archived off archive.apache.org - both downloadable
assert versions["5.0.2"].artifacts[0].downloadable is True
- assert versions["4.1.7"].artifacts[0].downloadable is False
+ assert versions["4.1.7"].artifacts[0].downloadable is True
assert versions["4.1.7"].svn_revision == 28114
@@ -137,7 +138,7 @@ def test_status_reflects_release_and_archive_state() -> None:
def test_archived_status_holds_without_an_archive_date() -> None:
# A release known to be archived but with no date (a catalogued historical release) carries
- # is_archived alone; it still reads as archived and isn't downloadable
+ # is_archived alone; it still reads as archived, and stays downloadable off archive.apache.org
project = _project()
undated = _release(project, "3.11.0", cycle_key="cassandra-default", released=_NOW, is_archived=True)
artifacts = [_artifact(project, "3.11.0", "a-3.11.0.tar.gz", release=undated)]
@@ -145,7 +146,7 @@ def test_archived_status_holds_without_an_archive_date() -> None:
versions = {str(v.version): v for v in catalog._versions(artifacts, {}, project.committee_key)}
assert versions["3.11.0"].status == "archived"
- assert versions["3.11.0"].artifacts[0].downloadable is False
+ assert versions["3.11.0"].artifacts[0].downloadable is True
def test_svn_revision_alone_does_not_make_a_version_managed() -> None:
diff --git a/tests/unit/test_decompose.py b/tests/unit/test_decompose.py
index ff14dfc8..04a4c6af 100644
--- a/tests/unit/test_decompose.py
+++ b/tests/unit/test_decompose.py
@@ -198,3 +198,12 @@ def test_repos_subtree_is_not_a_release() -> None:
"hadoop-0.20.205.0.2-1.x86_64.rpm",
)
assert result is None
+
+
+def test_candidate_keys_bridge_underscore_package_names() -> None:
+ # A dist filename's underscore package name lines up with the registry's hyphenated key, so
+ # the airflow providers resolve to airflow-providers-<name> rather than seeding a duplicate
+ assert "airflow-providers-alibaba" in dist.candidate_keys("airflow", "apache_airflow_providers_alibaba")
+ assert "airflow-providers" in dist.candidate_keys("airflow", "apache_airflow_providers")
+ # An apache- prefixed name still strips to the bare key, as before
+ assert dist.candidate_keys("cassandra", "apache-cassandra")[0] == "cassandra"
diff --git a/tests/unit/test_svn_publish_helpers.py b/tests/unit/test_svn_publish_helpers.py
index a93effab..66f2d633 100644
--- a/tests/unit/test_svn_publish_helpers.py
+++ b/tests/unit/test_svn_publish_helpers.py
@@ -96,6 +96,23 @@ def test_public_download_url_falls_back_to_self_host_without_svn(monkeypatch: py
assert url == "https://atr.example/downloads/apple/apple-1.0/apple-1.0.tar.gz"
+def test_public_download_url_uses_archive_for_archived_release(monkeypatch: pytest.MonkeyPatch) -> None:
+ # An archived release lives on archive.apache.org on every host, so even with no SVN publishing
+ # configured (dev) it short-circuits the self-hosted fallback rather than using it
+ monkeypatch.setattr(config.get(), "SVN_PUBLISH_URL", None)
+ committee = sql.Committee(key="apple", name="Apple", is_podling=False)
+
+ url = util.public_download_url(
+ committee,
+ safe.RelPath("apple-1.0"),
+ util.DownloadFile.ARTIFACT,
+ filename="apple-1.0.tar.gz",
+ archived=True,
+ )
+
+ assert url == f"{constants.ARCHIVE_APACHE_URL}/apple/apple-1.0/apple-1.0.tar.gz"
+
+
def test_public_download_url_uses_canonical_host_for_released_metadata(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(config.get(), "SVN_PUBLISH_URL", "https://internal.example.invalid/repos/dist/release")
monkeypatch.setattr(constants, "SVN_DIST_PUBLIC_URL", "https://dist.apache.org/repos/dist/release")
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org