You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tooling.apache.org by sb...@apache.org on 2026/09/03 18:03:07 UTC
(tooling-trusted-releases) 01/03: #1517 - break purls down with more classifiers
This is an automated email from the ASF dual-hosted git repository.
sbp pushed a commit to branch sbp
in repository https://gitbox.apache.org/repos/asf/tooling-trusted-releases.git
commit d4f75423b512c76574749fd7d8617dd42a1f8ae5
Author: Alastair McFarlane <ar...@apache.org>
AuthorDate: Wed Sep 2 14:36:22 2026 +0100
#1517 - break purls down with more classifiers
---
atr/catalog_site.py | 286 +++++++++++++++++++++-------
atr/classify.py | 127 ++++++++-----
tests/unit/test_catalog_site.py | 405 ++++++++++++++++++++++++++++++++++++++++
3 files changed, 701 insertions(+), 117 deletions(-)
diff --git a/atr/catalog_site.py b/atr/catalog_site.py
index 5134eb74..bd1e4ba0 100644
--- a/atr/catalog_site.py
+++ b/atr/catalog_site.py
@@ -26,14 +26,16 @@ only adds the file layout and the rendering.
"""
import asyncio
+import collections
import dataclasses
import datetime
+import itertools
import json
import pathlib
import re
import shutil
import urllib.parse
-from collections.abc import Awaitable, Container, Sequence
+from collections.abc import Awaitable, Container, Iterable, Iterator, Sequence
from typing import Any, Final
import aiofiles.os
@@ -41,6 +43,7 @@ import aioshutil
import jinja2
import sqlmodel
+import atr.classify as classify
import atr.cle as cle
import atr.constants as constants
import atr.db as db
@@ -70,6 +73,8 @@ _ENVIRONMENT: Final = jinja2.Environment(
# The footer states the running ATR version, the same as the main site's footer.
_ENVIRONMENT.globals["version"] = metadata.version
_ENVIRONMENT.globals["commit"] = metadata.commit
+# The footer stamps the render time, so it's evaluated per page rather than at import.
+_ENVIRONMENT.globals["last_updated"] = lambda: datetime.datetime.now(datetime.UTC).strftime("%d %b %Y %H:%M UTC")
# ATR's own stylesheets and the certified badge, copied under the site's assets/
# so the pages carry the app's look with nothing resolving back to the server.
@@ -112,6 +117,28 @@ _ENVIRONMENT.globals["attic_key"] = _ATTIC_COMMITTEE_KEY
# pages are written once the walk has been through every committee.
_INDEXING_COMMITTEE_KEYS: Final[frozenset[str]] = frozenset({_ATTIC_COMMITTEE_KEY, _INCUBATOR_COMMITTEE_KEY})
+# The `class` qualifier abbreviates an artifact's stored classification to the short token
+# filenames use. Only the primary-artifact classes appear; sbom and metadata are companions.
+_CLASS_QUALIFIER: Final[dict[str, str]] = {
+ classify.FileType.SOURCE.value: "src",
+ classify.FileType.BINARY.value: "bin",
+ classify.FileType.DOCS.value: "docs",
+}
+
+
+@dataclasses.dataclass
+class _CommitteeSummary:
+ release_count: int
+ latest_date: datetime.datetime | None
+ project_names: list[str]
+
+
+@dataclasses.dataclass
+class _SubprojectSummary:
+ release_count: int
+ latest_date: datetime.datetime | None
+ has_archived: bool
+
async def generate_all(data: db.Session) -> None:
"""Rebuild every page, overwriting in place so the site stays servable throughout."""
@@ -204,18 +231,106 @@ async def regenerate_project(data: db.Session, project_key: str) -> None:
await _write_index_pages(data, site_dir, committees)
-@dataclasses.dataclass
-class _CommitteeSummary:
- release_count: int
- latest_date: datetime.datetime | None
- project_names: list[str]
+def _artifact_classifier_combos(
+ version: api.CatalogVersion, pmc: str, project: str
+) -> Iterator[tuple[dict[str, str], str]]:
+ """(minimal unique classifier combo, download URL) per downloadable artifact.
+ A file's classifiers are class/os/arch/ext; the combo is the smallest subset no other file in
+ the release shares, so `?<combo>` names it and no fewer qualifiers would. A file whose whole
+ classifier set another file also has - or which carries no classifiers - yields nothing;
+ file_name and subpath still address it. Larger combos come first, so an over-specified query
+ matches the most specific rule.
+ """
+ strip = {"apache", pmc, project}
+ release_version = str(version.version)
+ classified: list[tuple[dict[str, str], str]] = []
+ for artifact in version.artifacts:
+ url = artifact.artifact_url
+ if url is None:
+ continue
+ classifiers = _artifact_classifiers(artifact, release_version, strip)
+ if classifiers:
+ classified.append((classifiers, url))
+ combos: list[tuple[dict[str, str], str]] = []
+ for index, (classifiers, url) in enumerate(classified):
+ others = [other for i, (other, _) in enumerate(classified) if i != index]
+ combo = _minimal_combo(classifiers, others)
+ if combo is not None:
+ combos.append((combo, url))
+ combos.sort(key=lambda combo_url: len(combo_url[0]), reverse=True)
+ yield from combos
+
+
+def _artifact_classifiers(artifact: api.CatalogArtifact, version: str, strip: Iterable[str]) -> dict[str, str]:
+ """The classifier qualifiers a downloadable artifact carries: class, os, arch, ext.
+
+ `class` is the stored classification mapped to its short token; `os`/`arch` are read from
+ the subpath tail and `ext` from the file name, all canonicalised by classify.
+ """
+ classifiers: dict[str, str] = {}
+ if artifact.classification is not None:
+ class_token = _CLASS_QUALIFIER.get(artifact.classification)
+ if class_token is not None:
+ classifiers["class"] = class_token
+ tail = _artifact_subpath(artifact.artifact_path, version, strip)
+ if (os_value := classify.os_in(tail)) is not None:
+ classifiers["os"] = os_value
+ if (arch_value := classify.arch_in(tail)) is not None:
+ classifiers["arch"] = arch_value
+ if (ext_value := classify.ext_of(artifact.artifact_path)) is not None:
+ classifiers["ext"] = ext_value
+ return classifiers
+
+
+def _artifact_downloads(artifact: api.CatalogArtifact) -> Iterator[tuple[str, str]]:
+ """(file name, download URL) per file an artifact exposes: itself, signature, checksum, SBOM.
+
+ A pair with no URL - not downloadable, or the companion absent - is skipped.
+ """
+ pairs = (
+ (artifact.artifact_path, artifact.artifact_url),
+ (artifact.signature_path, artifact.signature_url),
+ (artifact.checksum_path, artifact.checksum_url),
+ (artifact.sbom_path, artifact.sbom_url),
+ )
+ for file_name, url in pairs:
+ if (file_name is not None) and (url is not None):
+ yield file_name, url
+
+
+def _artifact_subpath(file_name: str, version: str, strip: Iterable[str]) -> str:
+ """An artifact's distinguishing tail, for the `subpath` qualifier.
+
+ The part after the release version when the file name carries it
+ (apache-ivy-2.0.0-bin-with-deps.tar.gz -> bin-with-deps.tar.gz); otherwise the file name
+ with `strip` (apache, the PMC and project names) removed longest-first
+ (apache-airflow-providers-amazon-1.0.0-bin.tar.gz -> amazon-1.0.0-bin.tar.gz).
+ """
+ index = file_name.find(version)
+ if index != -1:
+ return file_name[index + len(version) :].lstrip("-+._")
+ trimmed = file_name
+ for token in sorted(strip, key=len, reverse=True):
+ if token:
+ trimmed = re.sub(re.escape(token), "", trimmed, flags=re.IGNORECASE)
+ return trimmed.lstrip("-+._")
-@dataclasses.dataclass
-class _SubprojectSummary:
- release_count: int
- latest_date: datetime.datetime | None
- has_archived: bool
+
+def _artifact_subpaths(version: api.CatalogVersion, pmc: str, project: str) -> Iterator[tuple[str, str]]:
+ """(subpath, download URL) per downloadable artifact, skipping empty or colliding tails.
+
+ A tail two artifacts share can't name one file, so it's dropped; file_name still covers them.
+ """
+ release_version = str(version.version)
+ strip = {"apache", pmc, project}
+ derived: list[tuple[str, str]] = []
+ for artifact in version.artifacts:
+ url = artifact.artifact_url
+ if url is None:
+ continue
+ derived.append((_artifact_subpath(artifact.artifact_path, release_version, strip), url))
+ yield from _unique(derived)
async def _committee_summaries(data: db.Session, committees: Sequence[sql.Committee]) -> dict[str, _CommitteeSummary]:
@@ -250,15 +365,29 @@ async def _committee_summaries(data: db.Session, committees: Sequence[sql.Commit
}
+async def _guarded_write(description: str, work: Awaitable[None]) -> None:
+ """Run one index write on its own, logging and swallowing a failure so the rest proceed."""
+ try:
+ await work
+ except Exception:
+ log.exception(f"Failed to render catalog site {description}")
+
+
def _has_live_project(committee: sql.Committee) -> bool:
return any(project.status in _LIVE_PROJECT_STATUSES for project in committee.projects)
-def _last_updated() -> str:
- return datetime.datetime.now(datetime.UTC).strftime("%d %b %Y %H:%M UTC")
+def _htaccess_cond(qualifier: str, value: str) -> str:
+ """A RewriteCond matching `<qualifier>=<value>` anywhere in the query string."""
+ # Match the raw (percent-encoded) query string, regex-escaped so its metacharacters
+ # read literally.
+ encoded = re.escape(urllib.parse.quote(value, safe=""))
+ return f'RewriteCond %{{QUERY_STRING}} "(^|&){qualifier}={encoded}(&|$)"'
-_ENVIRONMENT.globals["last_updated"] = _last_updated
+def _htaccess_rule(qualifier: str, value: str, url: str) -> tuple[str, str]:
+ """One qualifier's RewriteCond/RewriteRule pair, redirecting `?<qualifier>=<value>`."""
+ return _htaccess_cond(qualifier, value), _redirect_rule(url)
async def _lifecycle_events(data: db.Session, project: sql.Project) -> Sequence[sql.LifecycleEvent]:
@@ -267,6 +396,16 @@ async def _lifecycle_events(data: db.Session, project: sql.Project) -> Sequence[
return (await data.execute(stmt)).scalars().all()
+def _minimal_combo(classifiers: dict[str, str], others: list[dict[str, str]]) -> dict[str, str] | None:
+ """The smallest subset of `classifiers` that no dict in `others` contains, or None."""
+ items = sorted(classifiers.items())
+ for size in range(1, len(items) + 1):
+ for subset in itertools.combinations(items, size):
+ if not any(all(other.get(key) == value for key, value in subset) for other in others):
+ return dict(subset)
+ return None
+
+
async def _prune_directories(directory: safe.StatePath, keep: Container[str]) -> None:
"""Remove the subdirectories of one that this build didn't write.
@@ -288,6 +427,35 @@ async def _prune_directories(directory: safe.StatePath, keep: Container[str]) ->
log.info(f"Catalog site: pruned {len(pruned)} stale directories from {path}: {', '.join(pruned)}")
+def _redirect_rule(url: str) -> str:
+ """The RewriteRule redirecting the release directory to a download URL."""
+ # archive.apache.org is a permanent home (301); the closer.lua mirror is not (302). NE
+ # stops Apache re-encoding the already percent-encoded URL.
+ status = 301 if url.startswith(constants.ARCHIVE_APACHE_URL) else 302
+ return f'RewriteRule "^$" "{url}" [R={status},NE,L]'
+
+
+def _release_htaccess(version: api.CatalogVersion, pmc: str, project: str) -> str | None:
+ """Render a release's files to an Apache `.htaccess` map, or None if none are downloadable.
+
+ The vhost routes a qualified PURL request into the release directory; these rules match its
+ query string and redirect to the download URL. Every file is reachable by `?file_name=`, each
+ artifact also by `?subpath=` and by the minimal combo of its class/os/arch/ext classifiers.
+ """
+ rules: list[str] = []
+ for artifact in version.artifacts:
+ for file_name, url in _artifact_downloads(artifact):
+ rules.extend(_htaccess_rule("file_name", file_name, url))
+ for subpath, url in _artifact_subpaths(version, pmc, project):
+ rules.extend(_htaccess_rule("subpath", subpath, url))
+ for combo, url in _artifact_classifier_combos(version, pmc, project):
+ rules.extend(_htaccess_cond(key, combo[key]) for key in sorted(combo))
+ rules.append(_redirect_rule(url))
+ if not rules:
+ return None
+ return "RewriteEngine On\n" + "\n".join(rules) + "\n"
+
+
async def _subproject_summaries(data: db.Session, subprojects: Sequence[sql.Project]) -> dict[str, _SubprojectSummary]:
return {subproject.key: await _subproject_summary(data, subproject) for subproject in subprojects}
@@ -308,6 +476,18 @@ async def _subproject_summary(data: db.Session, project: sql.Project) -> _Subpro
return _SubprojectSummary(release_count=release_count, latest_date=latest_date, has_archived=has_archived)
+def _unique(pairs: list[tuple[str, str]]) -> Iterator[tuple[str, str]]:
+ """Yield the pairs whose (non-empty) key is unique in the list.
+
+ A qualifier value that more than one artifact derives can't address a single file, so it's
+ dropped rather than resolving ambiguously; file_name always covers what's dropped.
+ """
+ counts = collections.Counter(key for key, _ in pairs)
+ for key, value in pairs:
+ if key and (counts[key] == 1):
+ yield key, value
+
+
async def _write(path: safe.StatePath, content: str) -> None:
await util.atomic_write_file(path.path, content)
@@ -398,6 +578,25 @@ async def _write_committee_pages(
return written
+async def _write_front_page(
+ data: db.Session,
+ site_dir: safe.StatePath,
+ all_committees: Sequence[sql.Committee],
+ listed: Sequence[sql.Committee],
+) -> None:
+ # Summaries cover every committee; only the still-releasing ones are listed.
+ summaries = await _committee_summaries(data, all_committees)
+ await _write_root_index(site_dir, listed, summaries)
+
+
+async def _write_htaccess(release_dir: safe.StatePath, version: api.CatalogVersion, pmc: str, project: str) -> None:
+ htaccess = _release_htaccess(version, pmc, project)
+ if htaccess is None:
+ return
+ # .htaccess is a dotfile, which StatePath's join rejects, so write via the raw OS path
+ await util.atomic_write_file(release_dir.path / ".htaccess", htaccess)
+
+
async def _write_incubator_index(
site_dir: safe.StatePath,
incubator: sql.Committee,
@@ -454,25 +653,6 @@ async def _write_index_pages(data: db.Session, site_dir: safe.StatePath, committ
await _guarded_write("front page", _write_front_page(data, site_dir, committees, current))
-async def _guarded_write(description: str, work: Awaitable[None]) -> None:
- """Run one index write on its own, logging and swallowing a failure so the rest proceed."""
- try:
- await work
- except Exception:
- log.exception(f"Failed to render catalog site {description}")
-
-
-async def _write_front_page(
- data: db.Session,
- site_dir: safe.StatePath,
- all_committees: Sequence[sql.Committee],
- listed: Sequence[sql.Committee],
-) -> None:
- # Summaries cover every committee; only the still-releasing ones are listed.
- summaries = await _committee_summaries(data, all_committees)
- await _write_root_index(site_dir, listed, summaries)
-
-
async def _write_project(
data: db.Session,
committee: sql.Committee,
@@ -562,45 +742,7 @@ async def _write_release(
),
)
await _write(release_dir / "artifacts.json", version.model_dump_json(indent=2))
- await _write_htaccess(release_dir, version)
-
-
-def _release_htaccess(version: api.CatalogVersion) -> str | None:
- """Render a release's artifacts to an Apache `.htaccess` qualifier map (#1517).
-
- A PURL names a single artifact with the `file_name` qualifier, which reaches us as
- the query string - a `#subpath` fragment never leaves the browser, a query string
- does. The vhost hands any qualified request into the release directory, where each
- downloadable artifact gets a rule matching `?file_name=<its name>` and redirecting
- to the file's real download URL. A bare request goes straight to artifacts.json and
- never reaches this file. Returns None when nothing in the release is downloadable.
- """
- lines = ["RewriteEngine On"]
- for artifact in version.artifacts:
- if artifact.artifact_url is None:
- continue
- # archive.a.o is a permanent archive, so a 301; anything else
- # will eventually move to archive, so don't let it be cached
- archived = artifact.artifact_url.startswith(constants.ARCHIVE_APACHE_URL)
- status = 301 if archived else 302
- # The file_name value arrives percent-encoded in the query string, so match that
- # encoded form, escaped so its metacharacters read literally. NE keeps the already
- # percent-encoded download URL from being escaped a second time.
- value = re.escape(urllib.parse.quote(artifact.artifact_path, safe=""))
- lines.append(f'RewriteCond %{{QUERY_STRING}} "(^|&)file_name={value}(&|$)"')
- lines.append(f'RewriteRule "^$" "{artifact.artifact_url}" [R={status},NE,L]')
- if len(lines) == 1:
- return None
- return "\n".join(lines) + "\n"
-
-
-async def _write_htaccess(release_dir: safe.StatePath, version: api.CatalogVersion) -> None:
- htaccess = _release_htaccess(version)
- if htaccess is None:
- return
- # RelPath refuses dotfiles, so this drops to the raw OS path for the fixed .htaccess name
- # rather than routing it through StatePath's validating join.
- await util.atomic_write_file(release_dir.path / ".htaccess", htaccess)
+ await _write_htaccess(release_dir, version, committee.key, project.key)
async def _write_root_index(
diff --git a/atr/classify.py b/atr/classify.py
index 7e94ba9e..a80f8ac7 100644
--- a/atr/classify.py
+++ b/atr/classify.py
@@ -63,28 +63,35 @@ _BIN_DISTRIBUTION_TOKENS: Final[frozenset[str]] = frozenset(
_BIN_PATH_PARTS: Final[frozenset[str]] = frozenset({"bin", "binaries", "binary"})
# 0.9% (path), 28.2% (filename)
-_BIN_PLATFORM_TOKENS: Final[frozenset[str]] = frozenset(
- {
- "64bit",
- "aarch64",
- "amd64",
- "apk",
- "arm64",
- "arm64bit",
- "darwin",
- "linux",
- "mac",
- "macos",
- "osx",
- "win",
- "windows",
- "win32",
- "win64",
- "x64",
- "x86",
- "x86_64",
- }
-)
+# OS and CPU-architecture tokens marking a binary distribution, as {token: canonical} so a
+# caller can read which OS or arch a filename declares (os_in/arch_in). Both separator
+# spellings of a compound arch are listed (x86-64/x86_64) since filenames use either;
+# classify only tests membership, taken from the union below.
+_OS_TOKENS: Final[dict[str, str]] = {
+ "darwin": "macos",
+ "mac": "macos",
+ "macos": "macos",
+ "osx": "macos",
+ "linux": "linux",
+ "win": "windows",
+ "win32": "windows",
+ "win64": "windows",
+ "windows": "windows",
+}
+_ARCH_TOKENS: Final[dict[str, str]] = {
+ "aarch64": "aarch64",
+ "arm64": "aarch64",
+ "arm64bit": "aarch64",
+ "amd64": "x86-64",
+ "x64": "x86-64",
+ "x86-64": "x86-64",
+ "x86_64": "x86-64",
+ "x86": "x86",
+ "64bit": "64bit",
+}
+# Platform markers that name neither an OS nor an arch (a package format).
+_PLATFORM_OTHER_TOKENS: Final[frozenset[str]] = frozenset({"apk"})
+_BIN_PLATFORM_TOKENS: Final[frozenset[str]] = frozenset(_OS_TOKENS) | frozenset(_ARCH_TOKENS) | _PLATFORM_OTHER_TOKENS
# Docs markers
@@ -134,6 +141,11 @@ class FileType(enum.Enum):
SBOM = "sbom"
+def arch_in(name: str) -> str | None:
+ """The canonical CPU architecture a name declares, if any."""
+ return _platform_match(name, _ARCH_TOKENS)
+
+
def archive_marker_counts(stem: str, path: pathlib.PurePath) -> tuple[int, int, int]:
name = pathlib.PurePosixPath(stem).name
tokens = _get_stem_tokens(name)
@@ -153,29 +165,6 @@ def archive_marker_counts(stem: str, path: pathlib.PurePath) -> tuple[int, int,
return source_count, binary_count, docs_count
-def classify_path(path: pathlib.PurePath) -> FileType:
- # The path-only half of classify(): no policy matchers and no reading of file
- # content, so it works when all we have is a path (a find-ls dump, or an SVN
- # commit payload). The marker counts and tie-breaks are the same, so a path
- # classified here lands the same as it would through the full classify(), give
- # or take an SBOM that only its content would give away.
- name = path.name
- if (name in analysis.DISALLOWED_FILENAMES) or (path.suffix in analysis.DISALLOWED_SUFFIXES):
- return FileType.DISALLOWED
- if analysis.is_cyclonedx(name):
- return FileType.SBOM
- if analysis.is_sbom_metadata(name):
- return FileType.METADATA
- path_str = str(path)
- search = re.search(analysis.extension_pattern(), path_str)
- if search and search.group("metadata"):
- return FileType.METADATA
- if (not search) or (not search.group("artifact")):
- return FileType.BINARY
- stem = path_str[: search.start()]
- return classify_from_counts(path_str, *archive_marker_counts(stem, path))
-
-
async def classify(
path: safe.RelPath,
base_path: safe.StatePath | None = None,
@@ -230,6 +219,38 @@ def classify_from_counts(path_str: str, source_count: int, binary_count: int, do
return FileType.BINARY
+def classify_path(path: pathlib.PurePath) -> FileType:
+ # The path-only half of classify(): no policy matchers and no reading of file
+ # content, so it works when all we have is a path (a find-ls dump, or an SVN
+ # commit payload). The marker counts and tie-breaks are the same, so a path
+ # classified here lands the same as it would through the full classify(), give
+ # or take an SBOM that only its content would give away.
+ name = path.name
+ if (name in analysis.DISALLOWED_FILENAMES) or (path.suffix in analysis.DISALLOWED_SUFFIXES):
+ return FileType.DISALLOWED
+ if analysis.is_cyclonedx(name):
+ return FileType.SBOM
+ if analysis.is_sbom_metadata(name):
+ return FileType.METADATA
+ path_str = str(path)
+ search = re.search(analysis.extension_pattern(), path_str)
+ if search and search.group("metadata"):
+ return FileType.METADATA
+ if (not search) or (not search.group("artifact")):
+ return FileType.BINARY
+ stem = path_str[: search.start()]
+ return classify_from_counts(path_str, *archive_marker_counts(stem, path))
+
+
+def ext_of(name: str) -> str | None:
+ """The packaging extension of an artifact file name (tar.gz, zip, jar, ...), or None."""
+ match = re.search(analysis.extension_pattern(), name)
+ if match is None:
+ return None
+ suffix = match.group("artifact") or match.group("metadata_artifact")
+ return suffix.removeprefix(".") if suffix else None
+
+
def matchers_from_policy(
source_artifact_paths: list[str],
binary_artifact_paths: list[str],
@@ -242,6 +263,11 @@ def matchers_from_policy(
return source_matcher, binary_matcher
+def os_in(name: str) -> str | None:
+ """The canonical operating system a name declares, if any."""
+ return _platform_match(name, _OS_TOKENS)
+
+
def _bin_binary_token(name: str) -> bool:
return bool(_BIN_BINARY_RE.search(name))
@@ -341,6 +367,17 @@ async def _npm_content_marker(archive_cache_dir: safe.StatePath) -> FileType | N
return FileType.BINARY
+def _platform_match(name: str, table: dict[str, str]) -> str | None:
+ # Match a vocab token delimited by the usual separators (or the ends), longest token first
+ # so a compound like x86-64 wins over x86. The name is a filename fragment, not a bare token,
+ # so darwin-aarch64 and MacOS_x86-64 both resolve however their axes are joined.
+ lower = name.lower()
+ for token in sorted(table, key=len, reverse=True):
+ if re.search(r"(^|[-_.])" + re.escape(token) + r"(?=[-_.]|$)", lower):
+ return table[token]
+ return None
+
+
async def _sbom_markers(path: safe.RelPath, base_path: safe.StatePath | None) -> FileType | None:
name = path.as_path().name
if analysis.is_cyclonedx(name):
diff --git a/tests/unit/test_catalog_site.py b/tests/unit/test_catalog_site.py
new file mode 100644
index 00000000..f01e0ff0
--- /dev/null
+++ b/tests/unit/test_catalog_site.py
@@ -0,0 +1,405 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import atr.catalog_site as catalog_site
+import atr.models.api as api
+import atr.models.safe as safe
+
+
+def _artifact(
+ path: str,
+ url: str | None,
+ *,
+ classification: str | None = None,
+ signature_path: str | None = None,
+ signature_url: str | None = None,
+ checksum_path: str | None = None,
+ checksum_url: str | None = None,
+ sbom_path: str | None = None,
+ sbom_url: str | None = None,
+) -> api.CatalogArtifact:
+ return api.CatalogArtifact(
+ artifact_path=path,
+ classification=classification,
+ signature_path=signature_path,
+ checksum_path=checksum_path,
+ sbom_path=sbom_path,
+ key_fingerprint=None,
+ svn_revision=None,
+ managed=False,
+ dated=None,
+ downloadable=url is not None,
+ artifact_url=url,
+ signature_url=signature_url,
+ checksum_url=checksum_url,
+ sbom_url=sbom_url,
+ )
+
+
+def _version(artifacts: list[api.CatalogArtifact], version: str = "1.0.0") -> api.CatalogVersion:
+ return api.CatalogVersion(
+ version=safe.VersionKey(version),
+ status="released",
+ released=None,
+ svn_revision=None,
+ managed=False,
+ cycle=None,
+ artifacts=artifacts,
+ )
+
+
+def test_htaccess_maps_each_downloadable_artifact_by_its_file_name_qualifier() -> None:
+ version = _version(
+ [
+ _artifact("apache-x-1.0.0-src.tgz", "https://mirror.example/x/apache-x-1.0.0-src.tgz?action=download"),
+ _artifact("apache-x-1.0.0-bin.zip", "https://mirror.example/x/apache-x-1.0.0-bin.zip?action=download"),
+ ]
+ )
+
+ htaccess = catalog_site._release_htaccess(version, "x", "x")
+
+ assert htaccess is not None
+ lines = htaccess.splitlines()
+ assert lines[0] == "RewriteEngine On"
+ assert 'RewriteCond %{QUERY_STRING} "(^|&)file_name=apache\\-x\\-1\\.0\\.0\\-src\\.tgz(&|$)"' in lines
+ assert 'RewriteRule "^$" "https://mirror.example/x/apache-x-1.0.0-src.tgz?action=download" [R=302,NE,L]' in lines
+ # RewriteEngine, then per artifact a Cond/Rule pair by file_name, by subpath, and by its
+ # unique ext (src.tgz vs bin.zip differ by extension)
+ assert len(lines) == 13
+
+
+def test_htaccess_addresses_the_artifact_by_the_tail_after_the_version() -> None:
+ # When the file name carries the release version, subpath is the tail after it - so
+ # apache-activemq-5.19.10-bin.tar.gz at 5.19.10 becomes bin.tar.gz.
+ version = _version(
+ [
+ _artifact(
+ "apache-activemq-5.19.10-bin.tar.gz",
+ "https://mirror.example/activemq/apache-activemq-5.19.10-bin.tar.gz?action=download",
+ )
+ ],
+ version="5.19.10",
+ )
+
+ htaccess = catalog_site._release_htaccess(version, "activemq", "activemq")
+
+ assert htaccess is not None
+ assert 'RewriteCond %{QUERY_STRING} "(^|&)subpath=bin\\.tar\\.gz(&|$)"' in htaccess
+
+
+def test_htaccess_subpath_strips_name_tokens_when_the_version_is_absent() -> None:
+ # A component bundle names files at the component's own version, so the release version
+ # (2020-12) isn't in the file name. subpath then strips apache and the PMC/project names,
+ # keeping the component name and its own version.
+ version = _version(
+ [
+ _artifact(
+ "apache-airflow-providers-amazon-1.0.0-bin.tar.gz",
+ "https://mirror.example/airflow/apache-airflow-providers-amazon-1.0.0-bin.tar.gz?action=download",
+ )
+ ],
+ version="2020-12",
+ )
+
+ htaccess = catalog_site._release_htaccess(version, "airflow", "airflow-providers")
+
+ assert htaccess is not None
+ assert 'RewriteCond %{QUERY_STRING} "(^|&)subpath=amazon\\-1\\.0\\.0\\-bin\\.tar\\.gz(&|$)"' in htaccess
+
+
+def test_htaccess_omits_subpath_when_two_artifacts_share_the_tail() -> None:
+ # Both files strip to the same tail (bin.tar.gz), so subpath can't name one - it's dropped
+ # and file_name still addresses each.
+ version = _version(
+ [
+ _artifact(
+ "apache-x-1.0.0-bin.tar.gz", "https://mirror.example/x/apache-x-1.0.0-bin.tar.gz?action=download"
+ ),
+ _artifact("x-1.0.0-bin.tar.gz", "https://mirror.example/x/x-1.0.0-bin.tar.gz?action=download"),
+ ]
+ )
+
+ htaccess = catalog_site._release_htaccess(version, "x", "x")
+
+ assert htaccess is not None
+ assert "subpath=" not in htaccess
+ assert htaccess.count("file_name=") == 2
+
+
+def test_htaccess_addresses_artifacts_by_their_class() -> None:
+ # The stored classification maps to a short class token: source -> src, binary -> bin.
+ version = _version(
+ [
+ _artifact(
+ "apache-x-1.0.0-src.tgz",
+ "https://mirror.example/x/apache-x-1.0.0-src.tgz?action=download",
+ classification="source",
+ ),
+ _artifact(
+ "apache-x-1.0.0-bin.zip",
+ "https://mirror.example/x/apache-x-1.0.0-bin.zip?action=download",
+ classification="binary",
+ ),
+ ]
+ )
+
+ htaccess = catalog_site._release_htaccess(version, "x", "x")
+
+ assert htaccess is not None
+ assert 'RewriteCond %{QUERY_STRING} "(^|&)class=src(&|$)"' in htaccess
+ assert 'RewriteCond %{QUERY_STRING} "(^|&)class=bin(&|$)"' in htaccess
+
+
+def test_htaccess_omits_classifiers_when_two_artifacts_share_all_of_them() -> None:
+ # Two source tarballs identical in every classifier (class and ext both match, no os/arch):
+ # no classifier combo can name one, so none is emitted - subpath and file_name still do.
+ version = _version(
+ [
+ _artifact(
+ "apache-x-1.0.0-src.tar.gz",
+ "https://mirror.example/x/apache-x-1.0.0-src.tar.gz?action=download",
+ classification="source",
+ ),
+ _artifact(
+ "apache-x-1.0.0-source.tar.gz",
+ "https://mirror.example/x/apache-x-1.0.0-source.tar.gz?action=download",
+ classification="source",
+ ),
+ ]
+ )
+
+ htaccess = catalog_site._release_htaccess(version, "x", "x")
+
+ assert htaccess is not None
+ assert "class=" not in htaccess
+ assert "ext=" not in htaccess
+ assert "subpath=" in htaccess
+
+
+def test_htaccess_combo_uses_os_alone_when_it_distinguishes() -> None:
+ # Two binaries differing only by OS: os alone is the minimal unique combo, so neither
+ # class (shared) nor arch (shared) appears in the rules.
+ version = _version(
+ [
+ _artifact(
+ "apache-x-1.0.0_macos_x86-64.tar.gz",
+ "https://m.example/x/macos.tar.gz?action=download",
+ classification="binary",
+ ),
+ _artifact(
+ "apache-x-1.0.0_linux_x86-64.tar.gz",
+ "https://m.example/x/linux.tar.gz?action=download",
+ classification="binary",
+ ),
+ ]
+ )
+
+ htaccess = catalog_site._release_htaccess(version, "x", "x")
+
+ assert htaccess is not None
+ assert 'RewriteCond %{QUERY_STRING} "(^|&)os=macos(&|$)"' in htaccess
+ assert 'RewriteCond %{QUERY_STRING} "(^|&)os=linux(&|$)"' in htaccess
+ assert "class=" not in htaccess
+ assert "arch=" not in htaccess
+
+
+def test_htaccess_combo_needs_os_and_arch_when_neither_alone_distinguishes() -> None:
+ # macos/x86-64 shares its OS with the aarch64 build and its arch with the linux build, so
+ # its minimal combo is os AND arch together, emitted as two conds on one rule.
+ version = _version(
+ [
+ _artifact(
+ "apache-x-1.0.0_macos_x86-64.tar.gz",
+ "https://m.example/x/mac-intel.tar.gz?action=download",
+ classification="binary",
+ ),
+ _artifact(
+ "apache-x-1.0.0_macos_aarch64.tar.gz",
+ "https://m.example/x/mac-arm.tar.gz?action=download",
+ classification="binary",
+ ),
+ _artifact(
+ "apache-x-1.0.0_linux_x86-64.tar.gz",
+ "https://m.example/x/linux.tar.gz?action=download",
+ classification="binary",
+ ),
+ ]
+ )
+
+ htaccess = catalog_site._release_htaccess(version, "x", "x")
+
+ assert htaccess is not None
+ assert (
+ 'RewriteCond %{QUERY_STRING} "(^|&)arch=x86\\-64(&|$)"\n'
+ 'RewriteCond %{QUERY_STRING} "(^|&)os=macos(&|$)"\n'
+ 'RewriteRule "^$" "https://m.example/x/mac-intel.tar.gz?action=download" [R=302,NE,L]'
+ ) in htaccess
+
+
+def test_htaccess_ext_distinguishes_format_twins() -> None:
+ # Same platform shipped as .tar.gz and .zip: class/os/arch are identical, so ext is the only
+ # classifier that separates them, and is the whole minimal combo here.
+ version = _version(
+ [
+ _artifact(
+ "apache-x-1.0.0-linux-amd64.tar.gz",
+ "https://m.example/x/l.tar.gz?action=download",
+ classification="binary",
+ ),
+ _artifact(
+ "apache-x-1.0.0-linux-amd64.zip", "https://m.example/x/l.zip?action=download", classification="binary"
+ ),
+ ]
+ )
+
+ htaccess = catalog_site._release_htaccess(version, "x", "x")
+
+ assert htaccess is not None
+ assert 'RewriteCond %{QUERY_STRING} "(^|&)ext=tar\\.gz(&|$)"' in htaccess
+ assert 'RewriteCond %{QUERY_STRING} "(^|&)ext=zip(&|$)"' in htaccess
+ assert "os=" not in htaccess
+ assert "arch=" not in htaccess
+
+
+def test_htaccess_addresses_signature_checksum_and_sbom_by_file_name() -> None:
+ version = _version(
+ [
+ _artifact(
+ "apache-x-1.0.0-src.tgz",
+ "https://mirror.example/x/apache-x-1.0.0-src.tgz?action=download",
+ signature_path="apache-x-1.0.0-src.tgz.asc",
+ signature_url="https://downloads.apache.org/x/apache-x-1.0.0-src.tgz.asc",
+ checksum_path="apache-x-1.0.0-src.tgz.sha512",
+ checksum_url="https://downloads.apache.org/x/apache-x-1.0.0-src.tgz.sha512",
+ sbom_path="apache-x-1.0.0-src.tgz.cdx.json",
+ sbom_url="https://downloads.apache.org/x/apache-x-1.0.0-src.tgz.cdx.json",
+ )
+ ]
+ )
+
+ htaccess = catalog_site._release_htaccess(version, "x", "x")
+
+ assert htaccess is not None
+ assert "file_name=apache\\-x\\-1\\.0\\.0\\-src\\.tgz\\.asc(&|$)" in htaccess
+ assert "file_name=apache\\-x\\-1\\.0\\.0\\-src\\.tgz\\.sha512(&|$)" in htaccess
+ assert "file_name=apache\\-x\\-1\\.0\\.0\\-src\\.tgz\\.cdx\\.json(&|$)" in htaccess
+ assert "https://downloads.apache.org/x/apache-x-1.0.0-src.tgz.asc" in htaccess
+ # RewriteEngine, a file_name pair for the artifact and each of its three companions, plus a
+ # subpath pair and an ext pair for the artifact
+ assert len(htaccess.splitlines()) == 13
+
+
+def test_htaccess_omits_companion_files_that_are_absent() -> None:
+ # An artifact with a signature but no checksum or SBOM gets file_name rules for just the two.
+ version = _version(
+ [
+ _artifact(
+ "apache-x-1.0.0-src.tgz",
+ "https://mirror.example/x/apache-x-1.0.0-src.tgz?action=download",
+ signature_path="apache-x-1.0.0-src.tgz.asc",
+ signature_url="https://downloads.apache.org/x/apache-x-1.0.0-src.tgz.asc",
+ )
+ ]
+ )
+
+ htaccess = catalog_site._release_htaccess(version, "x", "x")
+
+ assert htaccess is not None
+ assert ".asc" in htaccess
+ assert ".sha512" not in htaccess
+ # RewriteEngine, file_name pairs for the artifact and its signature, plus a subpath and an
+ # ext pair for the artifact
+ assert len(htaccess.splitlines()) == 9
+
+
+def test_htaccess_skips_artifacts_that_are_not_downloadable() -> None:
+ version = _version(
+ [
+ _artifact("apache-x-1.0.0-src.tgz", "https://mirror.example/x/apache-x-1.0.0-src.tgz?action=download"),
+ _artifact("apache-x-1.0.0-unpublished.tgz", None),
+ ]
+ )
+
+ htaccess = catalog_site._release_htaccess(version, "x", "x")
+
+ assert htaccess is not None
+ assert "unpublished" not in htaccess
+ # RewriteEngine, then the surviving artifact's file_name, subpath and ext pairs
+ assert len(htaccess.splitlines()) == 7
+
+
+def test_htaccess_is_omitted_when_nothing_is_downloadable() -> None:
+ version = _version([_artifact("apache-x-1.0.0-src.tgz", None)])
+
+ assert catalog_site._release_htaccess(version, "x", "x") is None
+
+
+def test_htaccess_redirect_is_temporary_for_the_mirror_and_permanent_for_the_archive() -> None:
+ # closer.lua is a mirror redirector that mustn't be cached as canonical (302); an
+ # archive.apache.org URL is a permanent home (301). The status keys off the target host.
+ version = _version(
+ [
+ _artifact(
+ "live-1.0.0-src.tgz", "https://www.apache.org/dyn/closer.lua/x/live-1.0.0-src.tgz?action=download"
+ ),
+ _artifact("old-1.0.0-src.tgz", "https://archive.apache.org/dist/x/old-1.0.0-src.tgz"),
+ ]
+ )
+
+ htaccess = catalog_site._release_htaccess(version, "x", "x")
+
+ assert htaccess is not None
+ rules = [line for line in htaccess.splitlines() if line.startswith("RewriteRule")]
+ live = next(line for line in rules if "live-1.0.0" in line)
+ old = next(line for line in rules if "old-1.0.0" in line)
+ assert "[R=302,NE,L]" in live
+ assert "[R=301,NE,L]" in old
+
+
+def test_htaccess_percent_encodes_and_escapes_the_file_name_value() -> None:
+ # The file_name value arrives percent-encoded in the query string (a C# hash as %23),
+ # and it's a PCRE pattern, so its dots must not read as wildcards.
+ version = _version(
+ [_artifact("mono-2.10.0-C#.zip", "https://mirror.example/x/mono-2.10.0-C%23.zip?action=download")]
+ )
+
+ htaccess = catalog_site._release_htaccess(version, "x", "x")
+
+ assert htaccess is not None
+ assert "file_name=mono\\-2\\.10\\.0\\-C%23\\.zip(&|$)" in htaccess
+
+
+async def test_write_htaccess_lands_the_dotfile_on_disk(tmp_path) -> None:
+ # .htaccess is a dotfile, which StatePath's validating join refuses; the write must drop
+ # to the raw OS path. This guards the file reaching disk, not just the generated string.
+ version = _version(
+ [_artifact("apache-x-1.0.0-src.tgz", "https://mirror.example/x/apache-x-1.0.0-src.tgz?action=download")]
+ )
+
+ await catalog_site._write_htaccess(safe.StatePath(tmp_path), version, "x", "x")
+
+ written = tmp_path / ".htaccess"
+ assert written.is_file()
+ assert written.read_text().startswith("RewriteEngine On")
+
+
+async def test_write_htaccess_writes_nothing_when_no_artifact_is_downloadable(tmp_path) -> None:
+ version = _version([_artifact("apache-x-1.0.0-src.tgz", None)])
+
+ await catalog_site._write_htaccess(safe.StatePath(tmp_path), version, "x", "x")
+
+ assert not (tmp_path / ".htaccess").exists()
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org