You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tooling.apache.org by ar...@apache.org on 2026/09/02 13:36:54 UTC
(tooling-trusted-releases) branch new-classifiers created (now 24b50978)
This is an automated email from the ASF dual-hosted git repository.
alitheg pushed a change to branch new-classifiers
in repository https://gitbox.apache.org/repos/asf/tooling-trusted-releases.git
at 24b50978 #1517 - break purls down with more classifiers
This branch includes the following new commits:
new 24b50978 #1517 - break purls down with more classifiers
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org
(tooling-trusted-releases) 01/01: #1517 - break purls down with more classifiers
Posted by ar...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
alitheg pushed a commit to branch new-classifiers
in repository https://gitbox.apache.org/repos/asf/tooling-trusted-releases.git
commit 24b50978b0000f6b07ab9b3457b4199528db2ec5
Author: Alastair McFarlane <ar...@apache.org>
AuthorDate: Wed Sep 2 14:36:22 2026 +0100
#1517 - break purls down with more classifiers
---
atr/catalog_site.py | 286 +++++++++++++++++++++-------
atr/classify.py | 127 ++++++++-----
tests/unit/test_catalog_site.py | 405 ++++++++++++++++++++++++++++++++++++++++
3 files changed, 701 insertions(+), 117 deletions(-)
diff --git a/atr/catalog_site.py b/atr/catalog_site.py
index 5134eb74..bd1e4ba0 100644
--- a/atr/catalog_site.py
+++ b/atr/catalog_site.py
@@ -26,14 +26,16 @@ only adds the file layout and the rendering.
"""
import asyncio
+import collections
import dataclasses
import datetime
+import itertools
import json
import pathlib
import re
import shutil
import urllib.parse
-from collections.abc import Awaitable, Container, Sequence
+from collections.abc import Awaitable, Container, Iterable, Iterator, Sequence
from typing import Any, Final
import aiofiles.os
@@ -41,6 +43,7 @@ import aioshutil
import jinja2
import sqlmodel
+import atr.classify as classify
import atr.cle as cle
import atr.constants as constants
import atr.db as db
@@ -70,6 +73,8 @@ _ENVIRONMENT: Final = jinja2.Environment(
# The footer states the running ATR version, the same as the main site's footer.
_ENVIRONMENT.globals["version"] = metadata.version
_ENVIRONMENT.globals["commit"] = metadata.commit
+# The footer stamps the render time, so it's evaluated per page rather than at import.
+_ENVIRONMENT.globals["last_updated"] = lambda: datetime.datetime.now(datetime.UTC).strftime("%d %b %Y %H:%M UTC")
# ATR's own stylesheets and the certified badge, copied under the site's assets/
# so the pages carry the app's look with nothing resolving back to the server.
@@ -112,6 +117,28 @@ _ENVIRONMENT.globals["attic_key"] = _ATTIC_COMMITTEE_KEY
# pages are written once the walk has been through every committee.
_INDEXING_COMMITTEE_KEYS: Final[frozenset[str]] = frozenset({_ATTIC_COMMITTEE_KEY, _INCUBATOR_COMMITTEE_KEY})
+# The `class` qualifier abbreviates an artifact's stored classification to the short token
+# filenames use. Only the primary-artifact classes appear; sbom and metadata are companions.
+_CLASS_QUALIFIER: Final[dict[str, str]] = {
+ classify.FileType.SOURCE.value: "src",
+ classify.FileType.BINARY.value: "bin",
+ classify.FileType.DOCS.value: "docs",
+}
+
+
+@dataclasses.dataclass
+class _CommitteeSummary:
+ release_count: int
+ latest_date: datetime.datetime | None
+ project_names: list[str]
+
+
+@dataclasses.dataclass
+class _SubprojectSummary:
+ release_count: int
+ latest_date: datetime.datetime | None
+ has_archived: bool
+
async def generate_all(data: db.Session) -> None:
"""Rebuild every page, overwriting in place so the site stays servable throughout."""
@@ -204,18 +231,106 @@ async def regenerate_project(data: db.Session, project_key: str) -> None:
await _write_index_pages(data, site_dir, committees)
-@dataclasses.dataclass
-class _CommitteeSummary:
- release_count: int
- latest_date: datetime.datetime | None
- project_names: list[str]
+def _artifact_classifier_combos(
+ version: api.CatalogVersion, pmc: str, project: str
+) -> Iterator[tuple[dict[str, str], str]]:
+ """(minimal unique classifier combo, download URL) per downloadable artifact.
+ A file's classifiers are class/os/arch/ext; the combo is the smallest subset no other file in
+ the release shares, so `?<combo>` names it and no fewer qualifiers would. A file whose whole
+ classifier set another file also has - or which carries no classifiers - yields nothing;
+ file_name and subpath still address it. Larger combos come first, so an over-specified query
+ matches the most specific rule.
+ """
+ strip = {"apache", pmc, project}
+ release_version = str(version.version)
+ classified: list[tuple[dict[str, str], str]] = []
+ for artifact in version.artifacts:
+ url = artifact.artifact_url
+ if url is None:
+ continue
+ classifiers = _artifact_classifiers(artifact, release_version, strip)
+ if classifiers:
+ classified.append((classifiers, url))
+ combos: list[tuple[dict[str, str], str]] = []
+ for index, (classifiers, url) in enumerate(classified):
+ others = [other for i, (other, _) in enumerate(classified) if i != index]
+ combo = _minimal_combo(classifiers, others)
+ if combo is not None:
+ combos.append((combo, url))
+ combos.sort(key=lambda combo_url: len(combo_url[0]), reverse=True)
+ yield from combos
+
+
+def _artifact_classifiers(artifact: api.CatalogArtifact, version: str, strip: Iterable[str]) -> dict[str, str]:
+ """The classifier qualifiers a downloadable artifact carries: class, os, arch, ext.
+
+ `class` is the stored classification mapped to its short token; `os`/`arch` are read from
+ the subpath tail and `ext` from the file name, all canonicalised by classify.
+ """
+ classifiers: dict[str, str] = {}
+ if artifact.classification is not None:
+ class_token = _CLASS_QUALIFIER.get(artifact.classification)
+ if class_token is not None:
+ classifiers["class"] = class_token
+ tail = _artifact_subpath(artifact.artifact_path, version, strip)
+ if (os_value := classify.os_in(tail)) is not None:
+ classifiers["os"] = os_value
+ if (arch_value := classify.arch_in(tail)) is not None:
+ classifiers["arch"] = arch_value
+ if (ext_value := classify.ext_of(artifact.artifact_path)) is not None:
+ classifiers["ext"] = ext_value
+ return classifiers
+
+
+def _artifact_downloads(artifact: api.CatalogArtifact) -> Iterator[tuple[str, str]]:
+ """(file name, download URL) per file an artifact exposes: itself, signature, checksum, SBOM.
+
+ A pair with no URL - not downloadable, or the companion absent - is skipped.
+ """
+ pairs = (
+ (artifact.artifact_path, artifact.artifact_url),
+ (artifact.signature_path, artifact.signature_url),
+ (artifact.checksum_path, artifact.checksum_url),
+ (artifact.sbom_path, artifact.sbom_url),
+ )
+ for file_name, url in pairs:
+ if (file_name is not None) and (url is not None):
+ yield file_name, url
+
+
+def _artifact_subpath(file_name: str, version: str, strip: Iterable[str]) -> str:
+ """An artifact's distinguishing tail, for the `subpath` qualifier.
+
+ The part after the release version when the file name carries it
+ (apache-ivy-2.0.0-bin-with-deps.tar.gz -> bin-with-deps.tar.gz); otherwise the file name
+ with `strip` (apache, the PMC and project names) removed longest-first
+ (apache-airflow-providers-amazon-1.0.0-bin.tar.gz -> amazon-1.0.0-bin.tar.gz).
+ """
+ index = file_name.find(version)
+ if index != -1:
+ return file_name[index + len(version) :].lstrip("-+._")
+ trimmed = file_name
+ for token in sorted(strip, key=len, reverse=True):
+ if token:
+ trimmed = re.sub(re.escape(token), "", trimmed, flags=re.IGNORECASE)
+ return trimmed.lstrip("-+._")
-@dataclasses.dataclass
-class _SubprojectSummary:
- release_count: int
- latest_date: datetime.datetime | None
- has_archived: bool
+
+def _artifact_subpaths(version: api.CatalogVersion, pmc: str, project: str) -> Iterator[tuple[str, str]]:
+ """(subpath, download URL) per downloadable artifact, skipping empty or colliding tails.
+
+ A tail two artifacts share can't name one file, so it's dropped; file_name still covers them.
+ """
+ release_version = str(version.version)
+ strip = {"apache", pmc, project}
+ derived: list[tuple[str, str]] = []
+ for artifact in version.artifacts:
+ url = artifact.artifact_url
+ if url is None:
+ continue
+ derived.append((_artifact_subpath(artifact.artifact_path, release_version, strip), url))
+ yield from _unique(derived)
async def _committee_summaries(data: db.Session, committees: Sequence[sql.Committee]) -> dict[str, _CommitteeSummary]:
@@ -250,15 +365,29 @@ async def _committee_summaries(data: db.Session, committees: Sequence[sql.Commit
}
+async def _guarded_write(description: str, work: Awaitable[None]) -> None:
+ """Run one index write on its own, logging and swallowing a failure so the rest proceed."""
+ try:
+ await work
+ except Exception:
+ log.exception(f"Failed to render catalog site {description}")
+
+
def _has_live_project(committee: sql.Committee) -> bool:
return any(project.status in _LIVE_PROJECT_STATUSES for project in committee.projects)
-def _last_updated() -> str:
- return datetime.datetime.now(datetime.UTC).strftime("%d %b %Y %H:%M UTC")
+def _htaccess_cond(qualifier: str, value: str) -> str:
+ """A RewriteCond matching `<qualifier>=<value>` anywhere in the query string."""
+ # Match the raw (percent-encoded) query string, regex-escaped so its metacharacters
+ # read literally.
+ encoded = re.escape(urllib.parse.quote(value, safe=""))
+ return f'RewriteCond %{{QUERY_STRING}} "(^|&){qualifier}={encoded}(&|$)"'
-_ENVIRONMENT.globals["last_updated"] = _last_updated
+def _htaccess_rule(qualifier: str, value: str, url: str) -> tuple[str, str]:
+ """One qualifier's RewriteCond/RewriteRule pair, redirecting `?<qualifier>=<value>`."""
+ return _htaccess_cond(qualifier, value), _redirect_rule(url)
async def _lifecycle_events(data: db.Session, project: sql.Project) -> Sequence[sql.LifecycleEvent]:
@@ -267,6 +396,16 @@ async def _lifecycle_events(data: db.Session, project: sql.Project) -> Sequence[
return (await data.execute(stmt)).scalars().all()
+def _minimal_combo(classifiers: dict[str, str], others: list[dict[str, str]]) -> dict[str, str] | None:
+ """The smallest subset of `classifiers` that no dict in `others` contains, or None."""
+ items = sorted(classifiers.items())
+ for size in range(1, len(items) + 1):
+ for subset in itertools.combinations(items, size):
+ if not any(all(other.get(key) == value for key, value in subset) for other in others):
+ return dict(subset)
+ return None
+
+
async def _prune_directories(directory: safe.StatePath, keep: Container[str]) -> None:
"""Remove the subdirectories of one that this build didn't write.
@@ -288,6 +427,35 @@ async def _prune_directories(directory: safe.StatePath, keep: Container[str]) ->
log.info(f"Catalog site: pruned {len(pruned)} stale directories from {path}: {', '.join(pruned)}")
+def _redirect_rule(url: str) -> str:
+ """The RewriteRule redirecting the release directory to a download URL."""
+ # archive.apache.org is a permanent home (301); the closer.lua mirror is not (302). NE
+ # stops Apache re-encoding the already percent-encoded URL.
+ status = 301 if url.startswith(constants.ARCHIVE_APACHE_URL) else 302
+ return f'RewriteRule "^$" "{url}" [R={status},NE,L]'
+
+
+def _release_htaccess(version: api.CatalogVersion, pmc: str, project: str) -> str | None:
+ """Render a release's files to an Apache `.htaccess` map, or None if none are downloadable.
+
+ The vhost routes a qualified PURL request into the release directory; these rules match its
+ query string and redirect to the download URL. Every file is reachable by `?file_name=`, each
+ artifact also by `?subpath=` and by the minimal combo of its class/os/arch/ext classifiers.
+ """
+ rules: list[str] = []
+ for artifact in version.artifacts:
+ for file_name, url in _artifact_downloads(artifact):
+ rules.extend(_htaccess_rule("file_name", file_name, url))
+ for subpath, url in _artifact_subpaths(version, pmc, project):
+ rules.extend(_htaccess_rule("subpath", subpath, url))
+ for combo, url in _artifact_classifier_combos(version, pmc, project):
+ rules.extend(_htaccess_cond(key, combo[key]) for key in sorted(combo))
+ rules.append(_redirect_rule(url))
+ if not rules:
+ return None
+ return "RewriteEngine On\n" + "\n".join(rules) + "\n"
+
+
async def _subproject_summaries(data: db.Session, subprojects: Sequence[sql.Project]) -> dict[str, _SubprojectSummary]:
return {subproject.key: await _subproject_summary(data, subproject) for subproject in subprojects}
@@ -308,6 +476,18 @@ async def _subproject_summary(data: db.Session, project: sql.Project) -> _Subpro
return _SubprojectSummary(release_count=release_count, latest_date=latest_date, has_archived=has_archived)
+def _unique(pairs: list[tuple[str, str]]) -> Iterator[tuple[str, str]]:
+ """Yield the pairs whose (non-empty) key is unique in the list.
+
+ A qualifier value that more than one artifact derives can't address a single file, so it's
+ dropped rather than resolving ambiguously; file_name always covers what's dropped.
+ """
+ counts = collections.Counter(key for key, _ in pairs)
+ for key, value in pairs:
+ if key and (counts[key] == 1):
+ yield key, value
+
+
async def _write(path: safe.StatePath, content: str) -> None:
await util.atomic_write_file(path.path, content)
@@ -398,6 +578,25 @@ async def _write_committee_pages(
return written
+async def _write_front_page(
+ data: db.Session,
+ site_dir: safe.StatePath,
+ all_committees: Sequence[sql.Committee],
+ listed: Sequence[sql.Committee],
+) -> None:
+ # Summaries cover every committee; only the still-releasing ones are listed.
+ summaries = await _committee_summaries(data, all_committees)
+ await _write_root_index(site_dir, listed, summaries)
+
+
+async def _write_htaccess(release_dir: safe.StatePath, version: api.CatalogVersion, pmc: str, project: str) -> None:
+ htaccess = _release_htaccess(version, pmc, project)
+ if htaccess is None:
+ return
+ # .htaccess is a dotfile, which StatePath's join rejects, so write via the raw OS path
+ await util.atomic_write_file(release_dir.path / ".htaccess", htaccess)
+
+
async def _write_incubator_index(
site_dir: safe.StatePath,
incubator: sql.Committee,
@@ -454,25 +653,6 @@ async def _write_index_pages(data: db.Session, site_dir: safe.StatePath, committ
await _guarded_write("front page", _write_front_page(data, site_dir, committees, current))
-async def _guarded_write(description: str, work: Awaitable[None]) -> None:
- """Run one index write on its own, logging and swallowing a failure so the rest proceed."""
- try:
- await work
- except Exception:
- log.exception(f"Failed to render catalog site {description}")
-
-
-async def _write_front_page(
- data: db.Session,
- site_dir: safe.StatePath,
- all_committees: Sequence[sql.Committee],
- listed: Sequence[sql.Committee],
-) -> None:
- # Summaries cover every committee; only the still-releasing ones are listed.
- summaries = await _committee_summaries(data, all_committees)
- await _write_root_index(site_dir, listed, summaries)
-
-
async def _write_project(
data: db.Session,
committee: sql.Committee,
@@ -562,45 +742,7 @@ async def _write_release(
),
)
await _write(release_dir / "artifacts.json", version.model_dump_json(indent=2))
- await _write_htaccess(release_dir, version)
-
-
-def _release_htaccess(version: api.CatalogVersion) -> str | None:
- """Render a release's artifacts to an Apache `.htaccess` qualifier map (#1517).
-
- A PURL names a single artifact with the `file_name` qualifier, which reaches us as
- the query string - a `#subpath` fragment never leaves the browser, a query string
- does. The vhost hands any qualified request into the release directory, where each
- downloadable artifact gets a rule matching `?file_name=<its name>` and redirecting
- to the file's real download URL. A bare request goes straight to artifacts.json and
- never reaches this file. Returns None when nothing in the release is downloadable.
- """
- lines = ["RewriteEngine On"]
- for artifact in version.artifacts:
- if artifact.artifact_url is None:
- continue
- # archive.a.o is a permanent archive, so a 301; anything else
- # will eventually move to archive, so don't let it be cached
- archived = artifact.artifact_url.startswith(constants.ARCHIVE_APACHE_URL)
- status = 301 if archived else 302
- # The file_name value arrives percent-encoded in the query string, so match that
- # encoded form, escaped so its metacharacters read literally. NE keeps the already
- # percent-encoded download URL from being escaped a second time.
- value = re.escape(urllib.parse.quote(artifact.artifact_path, safe=""))
- lines.append(f'RewriteCond %{{QUERY_STRING}} "(^|&)file_name={value}(&|$)"')
- lines.append(f'RewriteRule "^$" "{artifact.artifact_url}" [R={status},NE,L]')
- if len(lines) == 1:
- return None
- return "\n".join(lines) + "\n"
-
-
-async def _write_htaccess(release_dir: safe.StatePath, version: api.CatalogVersion) -> None:
- htaccess = _release_htaccess(version)
- if htaccess is None:
- return
- # RelPath refuses dotfiles, so this drops to the raw OS path for the fixed .htaccess name
- # rather than routing it through StatePath's validating join.
- await util.atomic_write_file(release_dir.path / ".htaccess", htaccess)
+ await _write_htaccess(release_dir, version, committee.key, project.key)
async def _write_root_index(
diff --git a/atr/classify.py b/atr/classify.py
index fae6c339..7fcab5cd 100644
--- a/atr/classify.py
+++ b/atr/classify.py
@@ -63,28 +63,35 @@ _BIN_DISTRIBUTION_TOKENS: Final[frozenset[str]] = frozenset(
_BIN_PATH_PARTS: Final[frozenset[str]] = frozenset({"bin", "binaries", "binary"})
# 0.9% (path), 28.2% (filename)
-_BIN_PLATFORM_TOKENS: Final[frozenset[str]] = frozenset(
- {
- "64bit",
- "aarch64",
- "amd64",
- "apk",
- "arm64",
- "arm64bit",
- "darwin",
- "linux",
- "mac",
- "macos",
- "osx",
- "win",
- "windows",
- "win32",
- "win64",
- "x64",
- "x86",
- "x86_64",
- }
-)
+# OS and CPU-architecture tokens marking a binary distribution, as {token: canonical} so a
+# caller can read which OS or arch a filename declares (os_in/arch_in). Both separator
+# spellings of a compound arch are listed (x86-64/x86_64) since filenames use either;
+# classify only tests membership, taken from the union below.
+_OS_TOKENS: Final[dict[str, str]] = {
+ "darwin": "macos",
+ "mac": "macos",
+ "macos": "macos",
+ "osx": "macos",
+ "linux": "linux",
+ "win": "windows",
+ "win32": "windows",
+ "win64": "windows",
+ "windows": "windows",
+}
+_ARCH_TOKENS: Final[dict[str, str]] = {
+ "aarch64": "aarch64",
+ "arm64": "aarch64",
+ "arm64bit": "aarch64",
+ "amd64": "x86-64",
+ "x64": "x86-64",
+ "x86-64": "x86-64",
+ "x86_64": "x86-64",
+ "x86": "x86",
+ "64bit": "64bit",
+}
+# Platform markers that name neither an OS nor an arch (a package format).
+_PLATFORM_OTHER_TOKENS: Final[frozenset[str]] = frozenset({"apk"})
+_BIN_PLATFORM_TOKENS: Final[frozenset[str]] = frozenset(_OS_TOKENS) | frozenset(_ARCH_TOKENS) | _PLATFORM_OTHER_TOKENS
# Docs markers
@@ -133,6 +140,11 @@ class FileType(enum.Enum):
SBOM = "sbom"
+def arch_in(name: str) -> str | None:
+ """The canonical CPU architecture a name declares, if any."""
+ return _platform_match(name, _ARCH_TOKENS)
+
+
def archive_marker_counts(stem: str, path: pathlib.PurePath) -> tuple[int, int, int]:
name = pathlib.PurePosixPath(stem).name
tokens = _get_stem_tokens(name)
@@ -152,29 +164,6 @@ def archive_marker_counts(stem: str, path: pathlib.PurePath) -> tuple[int, int,
return source_count, binary_count, docs_count
-def classify_path(path: pathlib.PurePath) -> FileType:
- # The path-only half of classify(): no policy matchers and no reading of file
- # content, so it works when all we have is a path (a find-ls dump, or an SVN
- # commit payload). The marker counts and tie-breaks are the same, so a path
- # classified here lands the same as it would through the full classify(), give
- # or take an SBOM that only its content would give away.
- name = path.name
- if (name in analysis.DISALLOWED_FILENAMES) or (path.suffix in analysis.DISALLOWED_SUFFIXES):
- return FileType.DISALLOWED
- if analysis.is_cyclonedx(name):
- return FileType.SBOM
- if analysis.is_sbom_metadata(name):
- return FileType.METADATA
- path_str = str(path)
- search = re.search(analysis.extension_pattern(), path_str)
- if search and search.group("metadata"):
- return FileType.METADATA
- if (not search) or (not search.group("artifact")):
- return FileType.BINARY
- stem = path_str[: search.start()]
- return classify_from_counts(path_str, *archive_marker_counts(stem, path))
-
-
async def classify(
path: safe.RelPath,
base_path: safe.StatePath | None = None,
@@ -229,6 +218,38 @@ def classify_from_counts(path_str: str, source_count: int, binary_count: int, do
return FileType.BINARY
+def classify_path(path: pathlib.PurePath) -> FileType:
+ # The path-only half of classify(): no policy matchers and no reading of file
+ # content, so it works when all we have is a path (a find-ls dump, or an SVN
+ # commit payload). The marker counts and tie-breaks are the same, so a path
+ # classified here lands the same as it would through the full classify(), give
+ # or take an SBOM that only its content would give away.
+ name = path.name
+ if (name in analysis.DISALLOWED_FILENAMES) or (path.suffix in analysis.DISALLOWED_SUFFIXES):
+ return FileType.DISALLOWED
+ if analysis.is_cyclonedx(name):
+ return FileType.SBOM
+ if analysis.is_sbom_metadata(name):
+ return FileType.METADATA
+ path_str = str(path)
+ search = re.search(analysis.extension_pattern(), path_str)
+ if search and search.group("metadata"):
+ return FileType.METADATA
+ if (not search) or (not search.group("artifact")):
+ return FileType.BINARY
+ stem = path_str[: search.start()]
+ return classify_from_counts(path_str, *archive_marker_counts(stem, path))
+
+
+def ext_of(name: str) -> str | None:
+ """The packaging extension of an artifact file name (tar.gz, zip, jar, ...), or None."""
+ match = re.search(analysis.extension_pattern(), name)
+ if match is None:
+ return None
+ suffix = match.group("artifact") or match.group("metadata_artifact")
+ return suffix.removeprefix(".") if suffix else None
+
+
def matchers_from_policy(
source_artifact_paths: list[str],
binary_artifact_paths: list[str],
@@ -241,6 +262,11 @@ def matchers_from_policy(
return source_matcher, binary_matcher
+def os_in(name: str) -> str | None:
+ """The canonical operating system a name declares, if any."""
+ return _platform_match(name, _OS_TOKENS)
+
+
def _bin_binary_token(name: str) -> bool:
return bool(_BIN_BINARY_RE.search(name))
@@ -340,6 +366,17 @@ async def _npm_content_marker(archive_cache_dir: safe.StatePath) -> FileType | N
return FileType.BINARY
+def _platform_match(name: str, table: dict[str, str]) -> str | None:
+ # Match a vocab token delimited by the usual separators (or the ends), longest token first
+ # so a compound like x86-64 wins over x86. The name is a filename fragment, not a bare token,
+ # so darwin-aarch64 and MacOS_x86-64 both resolve however their axes are joined.
+ lower = name.lower()
+ for token in sorted(table, key=len, reverse=True):
+ if re.search(r"(^|[-_.])" + re.escape(token) + r"(?=[-_.]|$)", lower):
+ return table[token]
+ return None
+
+
async def _sbom_markers(path: safe.RelPath, base_path: safe.StatePath | None) -> FileType | None:
name = path.as_path().name
if analysis.is_cyclonedx(name):
diff --git a/tests/unit/test_catalog_site.py b/tests/unit/test_catalog_site.py
new file mode 100644
index 00000000..f01e0ff0
--- /dev/null
+++ b/tests/unit/test_catalog_site.py
@@ -0,0 +1,405 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import atr.catalog_site as catalog_site
+import atr.models.api as api
+import atr.models.safe as safe
+
+
+def _artifact(
+ path: str,
+ url: str | None,
+ *,
+ classification: str | None = None,
+ signature_path: str | None = None,
+ signature_url: str | None = None,
+ checksum_path: str | None = None,
+ checksum_url: str | None = None,
+ sbom_path: str | None = None,
+ sbom_url: str | None = None,
+) -> api.CatalogArtifact:
+ return api.CatalogArtifact(
+ artifact_path=path,
+ classification=classification,
+ signature_path=signature_path,
+ checksum_path=checksum_path,
+ sbom_path=sbom_path,
+ key_fingerprint=None,
+ svn_revision=None,
+ managed=False,
+ dated=None,
+ downloadable=url is not None,
+ artifact_url=url,
+ signature_url=signature_url,
+ checksum_url=checksum_url,
+ sbom_url=sbom_url,
+ )
+
+
+def _version(artifacts: list[api.CatalogArtifact], version: str = "1.0.0") -> api.CatalogVersion:
+ return api.CatalogVersion(
+ version=safe.VersionKey(version),
+ status="released",
+ released=None,
+ svn_revision=None,
+ managed=False,
+ cycle=None,
+ artifacts=artifacts,
+ )
+
+
+def test_htaccess_maps_each_downloadable_artifact_by_its_file_name_qualifier() -> None:
+ version = _version(
+ [
+ _artifact("apache-x-1.0.0-src.tgz", "https://mirror.example/x/apache-x-1.0.0-src.tgz?action=download"),
+ _artifact("apache-x-1.0.0-bin.zip", "https://mirror.example/x/apache-x-1.0.0-bin.zip?action=download"),
+ ]
+ )
+
+ htaccess = catalog_site._release_htaccess(version, "x", "x")
+
+ assert htaccess is not None
+ lines = htaccess.splitlines()
+ assert lines[0] == "RewriteEngine On"
+ assert 'RewriteCond %{QUERY_STRING} "(^|&)file_name=apache\\-x\\-1\\.0\\.0\\-src\\.tgz(&|$)"' in lines
+ assert 'RewriteRule "^$" "https://mirror.example/x/apache-x-1.0.0-src.tgz?action=download" [R=302,NE,L]' in lines
+ # RewriteEngine, then per artifact a Cond/Rule pair by file_name, by subpath, and by its
+ # unique ext (src.tgz vs bin.zip differ by extension)
+ assert len(lines) == 13
+
+
+def test_htaccess_addresses_the_artifact_by_the_tail_after_the_version() -> None:
+ # When the file name carries the release version, subpath is the tail after it - so
+ # apache-activemq-5.19.10-bin.tar.gz at 5.19.10 becomes bin.tar.gz.
+ version = _version(
+ [
+ _artifact(
+ "apache-activemq-5.19.10-bin.tar.gz",
+ "https://mirror.example/activemq/apache-activemq-5.19.10-bin.tar.gz?action=download",
+ )
+ ],
+ version="5.19.10",
+ )
+
+ htaccess = catalog_site._release_htaccess(version, "activemq", "activemq")
+
+ assert htaccess is not None
+ assert 'RewriteCond %{QUERY_STRING} "(^|&)subpath=bin\\.tar\\.gz(&|$)"' in htaccess
+
+
+def test_htaccess_subpath_strips_name_tokens_when_the_version_is_absent() -> None:
+ # A component bundle names files at the component's own version, so the release version
+ # (2020-12) isn't in the file name. subpath then strips apache and the PMC/project names,
+ # keeping the component name and its own version.
+ version = _version(
+ [
+ _artifact(
+ "apache-airflow-providers-amazon-1.0.0-bin.tar.gz",
+ "https://mirror.example/airflow/apache-airflow-providers-amazon-1.0.0-bin.tar.gz?action=download",
+ )
+ ],
+ version="2020-12",
+ )
+
+ htaccess = catalog_site._release_htaccess(version, "airflow", "airflow-providers")
+
+ assert htaccess is not None
+ assert 'RewriteCond %{QUERY_STRING} "(^|&)subpath=amazon\\-1\\.0\\.0\\-bin\\.tar\\.gz(&|$)"' in htaccess
+
+
+def test_htaccess_omits_subpath_when_two_artifacts_share_the_tail() -> None:
+ # Both files strip to the same tail (bin.tar.gz), so subpath can't name one - it's dropped
+ # and file_name still addresses each.
+ version = _version(
+ [
+ _artifact(
+ "apache-x-1.0.0-bin.tar.gz", "https://mirror.example/x/apache-x-1.0.0-bin.tar.gz?action=download"
+ ),
+ _artifact("x-1.0.0-bin.tar.gz", "https://mirror.example/x/x-1.0.0-bin.tar.gz?action=download"),
+ ]
+ )
+
+ htaccess = catalog_site._release_htaccess(version, "x", "x")
+
+ assert htaccess is not None
+ assert "subpath=" not in htaccess
+ assert htaccess.count("file_name=") == 2
+
+
+def test_htaccess_addresses_artifacts_by_their_class() -> None:
+ # The stored classification maps to a short class token: source -> src, binary -> bin.
+ version = _version(
+ [
+ _artifact(
+ "apache-x-1.0.0-src.tgz",
+ "https://mirror.example/x/apache-x-1.0.0-src.tgz?action=download",
+ classification="source",
+ ),
+ _artifact(
+ "apache-x-1.0.0-bin.zip",
+ "https://mirror.example/x/apache-x-1.0.0-bin.zip?action=download",
+ classification="binary",
+ ),
+ ]
+ )
+
+ htaccess = catalog_site._release_htaccess(version, "x", "x")
+
+ assert htaccess is not None
+ assert 'RewriteCond %{QUERY_STRING} "(^|&)class=src(&|$)"' in htaccess
+ assert 'RewriteCond %{QUERY_STRING} "(^|&)class=bin(&|$)"' in htaccess
+
+
+def test_htaccess_omits_classifiers_when_two_artifacts_share_all_of_them() -> None:
+ # Two source tarballs identical in every classifier (class and ext both match, no os/arch):
+ # no classifier combo can name one, so none is emitted - subpath and file_name still do.
+ version = _version(
+ [
+ _artifact(
+ "apache-x-1.0.0-src.tar.gz",
+ "https://mirror.example/x/apache-x-1.0.0-src.tar.gz?action=download",
+ classification="source",
+ ),
+ _artifact(
+ "apache-x-1.0.0-source.tar.gz",
+ "https://mirror.example/x/apache-x-1.0.0-source.tar.gz?action=download",
+ classification="source",
+ ),
+ ]
+ )
+
+ htaccess = catalog_site._release_htaccess(version, "x", "x")
+
+ assert htaccess is not None
+ assert "class=" not in htaccess
+ assert "ext=" not in htaccess
+ assert "subpath=" in htaccess
+
+
+def test_htaccess_combo_uses_os_alone_when_it_distinguishes() -> None:
+ # Two binaries differing only by OS: os alone is the minimal unique combo, so neither
+ # class (shared) nor arch (shared) appears in the rules.
+ version = _version(
+ [
+ _artifact(
+ "apache-x-1.0.0_macos_x86-64.tar.gz",
+ "https://m.example/x/macos.tar.gz?action=download",
+ classification="binary",
+ ),
+ _artifact(
+ "apache-x-1.0.0_linux_x86-64.tar.gz",
+ "https://m.example/x/linux.tar.gz?action=download",
+ classification="binary",
+ ),
+ ]
+ )
+
+ htaccess = catalog_site._release_htaccess(version, "x", "x")
+
+ assert htaccess is not None
+ assert 'RewriteCond %{QUERY_STRING} "(^|&)os=macos(&|$)"' in htaccess
+ assert 'RewriteCond %{QUERY_STRING} "(^|&)os=linux(&|$)"' in htaccess
+ assert "class=" not in htaccess
+ assert "arch=" not in htaccess
+
+
+def test_htaccess_combo_needs_os_and_arch_when_neither_alone_distinguishes() -> None:
+ # macos/x86-64 shares its OS with the aarch64 build and its arch with the linux build, so
+ # its minimal combo is os AND arch together, emitted as two conds on one rule.
+ version = _version(
+ [
+ _artifact(
+ "apache-x-1.0.0_macos_x86-64.tar.gz",
+ "https://m.example/x/mac-intel.tar.gz?action=download",
+ classification="binary",
+ ),
+ _artifact(
+ "apache-x-1.0.0_macos_aarch64.tar.gz",
+ "https://m.example/x/mac-arm.tar.gz?action=download",
+ classification="binary",
+ ),
+ _artifact(
+ "apache-x-1.0.0_linux_x86-64.tar.gz",
+ "https://m.example/x/linux.tar.gz?action=download",
+ classification="binary",
+ ),
+ ]
+ )
+
+ htaccess = catalog_site._release_htaccess(version, "x", "x")
+
+ assert htaccess is not None
+ assert (
+ 'RewriteCond %{QUERY_STRING} "(^|&)arch=x86\\-64(&|$)"\n'
+ 'RewriteCond %{QUERY_STRING} "(^|&)os=macos(&|$)"\n'
+ 'RewriteRule "^$" "https://m.example/x/mac-intel.tar.gz?action=download" [R=302,NE,L]'
+ ) in htaccess
+
+
+def test_htaccess_ext_distinguishes_format_twins() -> None:
+ # Same platform shipped as .tar.gz and .zip: class/os/arch are identical, so ext is the only
+ # classifier that separates them, and is the whole minimal combo here.
+ version = _version(
+ [
+ _artifact(
+ "apache-x-1.0.0-linux-amd64.tar.gz",
+ "https://m.example/x/l.tar.gz?action=download",
+ classification="binary",
+ ),
+ _artifact(
+ "apache-x-1.0.0-linux-amd64.zip", "https://m.example/x/l.zip?action=download", classification="binary"
+ ),
+ ]
+ )
+
+ htaccess = catalog_site._release_htaccess(version, "x", "x")
+
+ assert htaccess is not None
+ assert 'RewriteCond %{QUERY_STRING} "(^|&)ext=tar\\.gz(&|$)"' in htaccess
+ assert 'RewriteCond %{QUERY_STRING} "(^|&)ext=zip(&|$)"' in htaccess
+ assert "os=" not in htaccess
+ assert "arch=" not in htaccess
+
+
+def test_htaccess_addresses_signature_checksum_and_sbom_by_file_name() -> None:
+ version = _version(
+ [
+ _artifact(
+ "apache-x-1.0.0-src.tgz",
+ "https://mirror.example/x/apache-x-1.0.0-src.tgz?action=download",
+ signature_path="apache-x-1.0.0-src.tgz.asc",
+ signature_url="https://downloads.apache.org/x/apache-x-1.0.0-src.tgz.asc",
+ checksum_path="apache-x-1.0.0-src.tgz.sha512",
+ checksum_url="https://downloads.apache.org/x/apache-x-1.0.0-src.tgz.sha512",
+ sbom_path="apache-x-1.0.0-src.tgz.cdx.json",
+ sbom_url="https://downloads.apache.org/x/apache-x-1.0.0-src.tgz.cdx.json",
+ )
+ ]
+ )
+
+ htaccess = catalog_site._release_htaccess(version, "x", "x")
+
+ assert htaccess is not None
+ assert "file_name=apache\\-x\\-1\\.0\\.0\\-src\\.tgz\\.asc(&|$)" in htaccess
+ assert "file_name=apache\\-x\\-1\\.0\\.0\\-src\\.tgz\\.sha512(&|$)" in htaccess
+ assert "file_name=apache\\-x\\-1\\.0\\.0\\-src\\.tgz\\.cdx\\.json(&|$)" in htaccess
+ assert "https://downloads.apache.org/x/apache-x-1.0.0-src.tgz.asc" in htaccess
+ # RewriteEngine, a file_name pair for the artifact and each of its three companions, plus a
+ # subpath pair and an ext pair for the artifact
+ assert len(htaccess.splitlines()) == 13
+
+
+def test_htaccess_omits_companion_files_that_are_absent() -> None:
+ # An artifact with a signature but no checksum or SBOM gets file_name rules for just the two.
+ version = _version(
+ [
+ _artifact(
+ "apache-x-1.0.0-src.tgz",
+ "https://mirror.example/x/apache-x-1.0.0-src.tgz?action=download",
+ signature_path="apache-x-1.0.0-src.tgz.asc",
+ signature_url="https://downloads.apache.org/x/apache-x-1.0.0-src.tgz.asc",
+ )
+ ]
+ )
+
+ htaccess = catalog_site._release_htaccess(version, "x", "x")
+
+ assert htaccess is not None
+ assert ".asc" in htaccess
+ assert ".sha512" not in htaccess
+ # RewriteEngine, file_name pairs for the artifact and its signature, plus a subpath and an
+ # ext pair for the artifact
+ assert len(htaccess.splitlines()) == 9
+
+
+def test_htaccess_skips_artifacts_that_are_not_downloadable() -> None:
+ version = _version(
+ [
+ _artifact("apache-x-1.0.0-src.tgz", "https://mirror.example/x/apache-x-1.0.0-src.tgz?action=download"),
+ _artifact("apache-x-1.0.0-unpublished.tgz", None),
+ ]
+ )
+
+ htaccess = catalog_site._release_htaccess(version, "x", "x")
+
+ assert htaccess is not None
+ assert "unpublished" not in htaccess
+ # RewriteEngine, then the surviving artifact's file_name, subpath and ext pairs
+ assert len(htaccess.splitlines()) == 7
+
+
+def test_htaccess_is_omitted_when_nothing_is_downloadable() -> None:
+ version = _version([_artifact("apache-x-1.0.0-src.tgz", None)])
+
+ assert catalog_site._release_htaccess(version, "x", "x") is None
+
+
+def test_htaccess_redirect_is_temporary_for_the_mirror_and_permanent_for_the_archive() -> None:
+ # closer.lua is a mirror redirector that mustn't be cached as canonical (302); an
+ # archive.apache.org URL is a permanent home (301). The status keys off the target host.
+ version = _version(
+ [
+ _artifact(
+ "live-1.0.0-src.tgz", "https://www.apache.org/dyn/closer.lua/x/live-1.0.0-src.tgz?action=download"
+ ),
+ _artifact("old-1.0.0-src.tgz", "https://archive.apache.org/dist/x/old-1.0.0-src.tgz"),
+ ]
+ )
+
+ htaccess = catalog_site._release_htaccess(version, "x", "x")
+
+ assert htaccess is not None
+ rules = [line for line in htaccess.splitlines() if line.startswith("RewriteRule")]
+ live = next(line for line in rules if "live-1.0.0" in line)
+ old = next(line for line in rules if "old-1.0.0" in line)
+ assert "[R=302,NE,L]" in live
+ assert "[R=301,NE,L]" in old
+
+
+def test_htaccess_percent_encodes_and_escapes_the_file_name_value() -> None:
+ # The file_name value arrives percent-encoded in the query string (a C# hash as %23),
+ # and it's a PCRE pattern, so its dots must not read as wildcards.
+ version = _version(
+ [_artifact("mono-2.10.0-C#.zip", "https://mirror.example/x/mono-2.10.0-C%23.zip?action=download")]
+ )
+
+ htaccess = catalog_site._release_htaccess(version, "x", "x")
+
+ assert htaccess is not None
+ assert "file_name=mono\\-2\\.10\\.0\\-C%23\\.zip(&|$)" in htaccess
+
+
+async def test_write_htaccess_lands_the_dotfile_on_disk(tmp_path) -> None:
+ # .htaccess is a dotfile, which StatePath's validating join refuses; the write must drop
+ # to the raw OS path. This guards the file reaching disk, not just the generated string.
+ version = _version(
+ [_artifact("apache-x-1.0.0-src.tgz", "https://mirror.example/x/apache-x-1.0.0-src.tgz?action=download")]
+ )
+
+ await catalog_site._write_htaccess(safe.StatePath(tmp_path), version, "x", "x")
+
+ written = tmp_path / ".htaccess"
+ assert written.is_file()
+ assert written.read_text().startswith("RewriteEngine On")
+
+
+async def test_write_htaccess_writes_nothing_when_no_artifact_is_downloadable(tmp_path) -> None:
+ version = _version([_artifact("apache-x-1.0.0-src.tgz", None)])
+
+ await catalog_site._write_htaccess(safe.StatePath(tmp_path), version, "x", "x")
+
+ assert not (tmp_path / ".htaccess").exists()
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org