You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tooling.apache.org by ar...@apache.org on 2026/09/08 14:34:21 UTC
(tooling-trusted-releases) branch arm updated: #1430 - RAT excludes URL
This is an automated email from the ASF dual-hosted git repository.
alitheg pushed a commit to branch arm
in repository https://gitbox.apache.org/repos/asf/tooling-trusted-releases.git
The following commit(s) were added to refs/heads/arm by this push:
new 9db69cb3 #1430 - RAT excludes URL
9db69cb3 is described below
commit 9db69cb36d8421b4b744c9c14e8e3f15d8a94545
Author: Alastair McFarlane <ar...@apache.org>
AuthorDate: Tue Sep 8 15:33:56 2026 +0100
#1430 - RAT excludes URL
---
atr/attestable.py | 35 +++++--
atr/docs/checks.md | 2 +
atr/get/projects.py | 2 +
atr/hashes.py | 6 ++
atr/models/api.py | 4 +
atr/models/attestable.py | 11 +++
atr/models/sql.py | 8 ++
atr/models/validation.py | 23 +++++
atr/shared/projects.py | 12 +++
atr/storage/writers/policy.py | 1 +
atr/tasks/checks/__init__.py | 19 ++++
atr/tasks/checks/rat.py | 119 ++++++++++++++++++++++--
migrations/versions/0125_2026.09.08_fbcb11d5.py | 44 +++++++++
tests/unit/test_asf_yaml_export.py | 14 +++
tests/unit/test_checks_rat.py | 74 +++++++++++++++
tests/unit/test_rat_excludes.py | 87 +++++++++++++++++
16 files changed, 447 insertions(+), 14 deletions(-)
diff --git a/atr/attestable.py b/atr/attestable.py
index 9cd1345b..abf10390 100644
--- a/atr/attestable.py
+++ b/atr/attestable.py
@@ -444,16 +444,37 @@ async def write_checks_data(
log.info(f"Writing checks for {project_key}/{version_key}/{revision_number}/{rel_path}: {checks}")
def modify(content: str) -> str:
+ # Load the whole document, not just .checks, so we don't drop a rat_excludes record
+ # written alongside it for the same revision
try:
- current = models.AttestableChecksV2.model_validate_json(content).checks
+ doc = models.AttestableChecksV2.model_validate_json(content)
except pydantic.ValidationError:
- current = {}
- if rel_path not in current:
- current[rel_path] = checks
+ doc = models.AttestableChecksV2()
+ if rel_path not in doc.checks:
+ doc.checks[rel_path] = checks
else:
- current[rel_path].update(checks)
- result = models.AttestableChecksV2(checks=current)
- return result.model_dump_json(indent=2)
+ doc.checks[rel_path].update(checks)
+ return doc.model_dump_json(indent=2)
+
+ await _atomic_modify_readonly(attestable_checks_path(project_key, version_key, revision_number).path, modify)
+
+
+async def write_rat_excludes_data(
+ project_key: safe.ProjectKey,
+ version_key: safe.VersionKey,
+ revision_number: safe.RevisionNumber,
+ rel_path: str,
+ record: models.RatExcludesRecordV2,
+) -> None:
+ log.info(f"Recording RAT excludes for {project_key}/{version_key}/{revision_number}/{rel_path}")
+
+ def modify(content: str) -> str:
+ try:
+ doc = models.AttestableChecksV2.model_validate_json(content)
+ except pydantic.ValidationError:
+ doc = models.AttestableChecksV2()
+ doc.rat_excludes[rel_path] = record
+ return doc.model_dump_json(indent=2)
await _atomic_modify_readonly(attestable_checks_path(project_key, version_key, revision_number).path, modify)
diff --git a/atr/docs/checks.md b/atr/docs/checks.md
index 4decd6e8..6404d656 100644
--- a/atr/docs/checks.md
+++ b/atr/docs/checks.md
@@ -126,6 +126,8 @@ You can [read more about license checks](license-checks).
Two separate sets of exclusion patterns let you skip files during license scanning. The _RAT source excludes_ are applied when RAT scans a source artifact that does not contain its own `.rat-excludes` file. The _Lightweight source excludes_ are always applied during the lightweight license header scan for source artifacts. In both cases the exclusions only take effect for artifacts that are classified as source by the source artifact paths setting (this is a [bug](https://github.com/apac [...]
+If you would rather not maintain the RAT excludes here as well as in your project, set a _RAT excludes URL_ pointing at a `.rat-excludes` file your project already keeps in git. We fetch it fresh for each revision we check, and record what we used against the revision. The URL must be on an `apache.org` host or `raw.githubusercontent.com`. When a source artifact ships its own `.rat-excludes` that still wins, then the URL, then the _RAT source excludes_ typed above.
+
You can [read more about license check exclusions](license-checks#project-policy-exclusions).
### Committee signing keys
diff --git a/atr/get/projects.py b/atr/get/projects.py
index 85fce92f..3ecfdfcf 100644
--- a/atr/get/projects.py
+++ b/atr/get/projects.py
@@ -476,6 +476,7 @@ def _asf_yaml_policy_fields(release_policy: sql.ReleasePolicy) -> dict[str, obje
"github_repository_name",
"github_repository_branch",
"download_path_suffix",
+ "rat_excludes_url",
):
value = getattr(release_policy, field)
if value:
@@ -850,6 +851,7 @@ async def _render_compose_form(project: sql.Project, *, readonly: bool) -> htm.E
"license_check_mode": project.policy_license_check_mode,
"source_excludes_lightweight": "\n".join(project.policy_source_excludes_lightweight),
"source_excludes_rat": "\n".join(project.policy_source_excludes_rat),
+ "rat_excludes_url": project.policy_rat_excludes_url,
"file_tag_mappings": atr_tag_yaml,
},
form_classes=".atr-canary.py-4.px-5",
diff --git a/atr/hashes.py b/atr/hashes.py
index 821c69a2..f5842442 100644
--- a/atr/hashes.py
+++ b/atr/hashes.py
@@ -33,6 +33,12 @@ def compute_dict_hash(to_hash: dict[Any, Any]) -> str:
return f"blake3:{hasher.hexdigest()}"
+def compute_bytes_hash(data: bytes) -> str:
+ hasher = blake3.blake3()
+ hasher.update(data)
+ return f"blake3:{hasher.hexdigest()}"
+
+
async def compute_file_hash(path: str | os.PathLike) -> str:
path = pathlib.Path(path)
hasher = blake3.blake3()
diff --git a/atr/models/api.py b/atr/models/api.py
index 4fd1ea53..f9db48b5 100644
--- a/atr/models/api.py
+++ b/atr/models/api.py
@@ -463,6 +463,7 @@ class PolicyArgsBase(schema.Strict):
source_artifact_paths: list[str] | None = None
source_excludes_lightweight: list[str] | None = None
source_excludes_rat: list[str] | None = None
+ rat_excludes_url: str | None = None
start_vote_subject: str | None = None
start_vote_template: str | None = None
finish_vote_template: str | None = None
@@ -500,6 +501,9 @@ class PolicyArgsBase(schema.Strict):
if self.download_path_suffix is not None:
validation.validate_download_path_suffix(self.download_path_suffix)
+ if self.rat_excludes_url is not None:
+ validation.validate_rat_excludes_url(self.rat_excludes_url)
+
github_repository_name = self.github_repository_name
if github_repository_name is not None:
validation.validate_github_repository_name(github_repository_name.strip())
diff --git a/atr/models/attestable.py b/atr/models/attestable.py
index 2ec7a44c..d85fb81a 100644
--- a/atr/models/attestable.py
+++ b/atr/models/attestable.py
@@ -94,9 +94,20 @@ class AttestableChecksV1(schema.Strict):
# Attestable Checks, v2
+class RatExcludesRecordV2(schema.Strict):
+ # What we fed to RAT for one source artifact. content and content_hash are set when the fetch
+ # succeeded; error is set instead when the URL was configured but couldn't be honoured
+ url: str
+ content_hash: str | None = None
+ content: str | None = None
+ error: str | None = None
+
+
class AttestableChecksV2(schema.Strict):
version: Literal[2] = 2
checks: dict[str, dict[str, str]] = schema.factory(dict)
+ # Keyed by the source artifact's rel_path
+ rat_excludes: dict[str, RatExcludesRecordV2] = schema.factory(dict)
# Attestable Checks, any version
diff --git a/atr/models/sql.py b/atr/models/sql.py
index b1fe1bc9..1780ad51 100644
--- a/atr/models/sql.py
+++ b/atr/models/sql.py
@@ -1394,6 +1394,12 @@ Sincerely,
return []
return policy.source_excludes_rat or []
+ @property
+ def policy_rat_excludes_url(self) -> str:
+ if (policy := self.release_policy) is None:
+ return ""
+ return policy.rat_excludes_url or ""
+
@property
def policy_tagging_spec(self) -> dict[str, Any] | None:
if (policy := self.release_policy) is None:
@@ -2247,6 +2253,7 @@ class ReleasePolicy(sqlmodel.SQLModel, table=True):
source_excludes_rat: list[str] = sqlmodel.Field(
default_factory=list, sa_column=sqlalchemy.Column(sqlalchemy.JSON, nullable=False)
)
+ rat_excludes_url: str = sqlmodel.Field(default="")
github_repository_name: str = sqlmodel.Field(default="")
github_repository_branch: str = sqlmodel.Field(default="")
github_compose_workflow_path: list[str] = sqlmodel.Field(
@@ -2286,6 +2293,7 @@ class ReleasePolicy(sqlmodel.SQLModel, table=True):
license_check_mode=self.license_check_mode,
source_excludes_lightweight=list(self.source_excludes_lightweight),
source_excludes_rat=list(self.source_excludes_rat),
+ rat_excludes_url=self.rat_excludes_url,
github_repository_name=self.github_repository_name,
github_repository_branch=self.github_repository_branch,
github_compose_workflow_path=list(self.github_compose_workflow_path),
diff --git a/atr/models/validation.py b/atr/models/validation.py
index c6c76578..fef2e0d0 100644
--- a/atr/models/validation.py
+++ b/atr/models/validation.py
@@ -35,6 +35,10 @@ REPOSITORY_URI_SCHEMES: Final[frozenset[str]] = frozenset(
)
STANDARD_URI_SCHEMES: Final[frozenset[str]] = frozenset({"http", "https"})
+# Non-apache.org hosts we'll fetch a project's .rat-excludes from. apache.org and its subdomains
+# are allowed by suffix in validate_rat_excludes_url, so only extra hosts need listing here
+RAT_EXCLUDES_URL_HOSTS: Final[frozenset[str]] = frozenset({"raw.githubusercontent.com"})
+
def compile_ignore_pattern(pattern: str):
# TODO: This requires importing re2 in atr/models
@@ -126,6 +130,25 @@ def validate_policy_min_hours(min_hours: int) -> None:
raise ValueError("Minimum voting period must be 0 or between 72 and 168 hours inclusive.")
+def validate_rat_excludes_url(url: str) -> None:
+ # We fetch this server-side on every checked revision, so an open URL would be an SSRF lever.
+ # Keep the hosts tight: apache.org covers gitbox, svn, and dist, and raw.githubusercontent.com
+ # covers projects that keep the file in a GitHub repo.
+ resolved = url.strip()
+ if not resolved:
+ return
+ if not uri_scheme_allowed(resolved, frozenset({"https"})):
+ raise ValueError("RAT excludes URL must use the https scheme.")
+ host = (parse.urlsplit(resolved).hostname or "").lower()
+ if not host:
+ raise ValueError("RAT excludes URL must include a host.")
+ apache = (host == "apache.org") or host.endswith(".apache.org")
+ if not (apache or (host in RAT_EXCLUDES_URL_HOSTS)):
+ raise ValueError(
+ f"RAT excludes URL host '{host}' is not allowed; use an apache.org host or raw.githubusercontent.com."
+ )
+
+
def validate_security_contact(committee_key: str, security_contact: str | None) -> None:
if not security_contact:
return
diff --git a/atr/shared/projects.py b/atr/shared/projects.py
index 3841a002..b6b53563 100644
--- a/atr/shared/projects.py
+++ b/atr/shared/projects.py
@@ -170,6 +170,12 @@ class ComposePolicyForm(form.Form):
widget=form.Widget.TEXTAREA,
rows=3,
)
+ rat_excludes_url: str = form.label(
+ "RAT excludes URL",
+ "URL of a .rat-excludes file we fetch fresh on each checked revision. Must be an"
+ " apache.org host or raw.githubusercontent.com.",
+ default="",
+ )
file_tag_mappings: str = form.label(
"Tagging spec",
"Spec for which files should be tagged for release in specific distribution types, YAML format",
@@ -177,6 +183,12 @@ class ComposePolicyForm(form.Form):
rows=3,
)
+ @pydantic.field_validator("rat_excludes_url", mode="before")
+ @classmethod
+ def validate_rat_excludes_url(cls, val: str) -> str:
+ validation.validate_rat_excludes_url(val)
+ return val
+
class VotePolicyForm(form.Form):
variant: VOTE = form.value(VOTE)
diff --git a/atr/storage/writers/policy.py b/atr/storage/writers/policy.py
index f34c8f78..2ab28d7a 100644
--- a/atr/storage/writers/policy.py
+++ b/atr/storage/writers/policy.py
@@ -152,6 +152,7 @@ class ReleaseManager(CommitteeParticipant):
release_policy.license_check_mode = license_check_mode
release_policy.source_excludes_lightweight = _split_lines_verbatim(form.source_excludes_lightweight)
release_policy.source_excludes_rat = _split_lines_verbatim(form.source_excludes_rat)
+ release_policy.rat_excludes_url = form.rat_excludes_url.strip()
release_policy.file_tag_mappings = atr_tags_dict
await self.__commit_and_log(str(project_key))
diff --git a/atr/tasks/checks/__init__.py b/atr/tasks/checks/__init__.py
index bed0abc4..c60e9d9b 100644
--- a/atr/tasks/checks/__init__.py
+++ b/atr/tasks/checks/__init__.py
@@ -41,6 +41,7 @@ import atr.log as log
import atr.models.safe as safe
import atr.models.sql as sql
import atr.paths as file_paths
+import atr.rat_excludes as rat_excludes
import atr.util as util
@@ -551,6 +552,23 @@ async def _resolve_is_podling(release: sql.Release, rel_path: str | None = None)
return (release.committee is not None) and release.committee.is_podling
+async def _resolve_rat_excludes_hash(release: sql.Release, rel_path: str | None = None) -> str:
+ policy = release.release_policy or release.project.release_policy
+ url = (policy.rat_excludes_url if (policy is not None) else "").strip()
+ if not url:
+ return ""
+ try:
+ content = await rat_excludes.fetch(url)
+ except rat_excludes.RatExcludesError:
+ # A failed fetch still has to bust any cached pass, and must stay distinct from the unset
+ # "" case, so we key on the URL rather than the content we couldn't get
+ return f"unavailable:{url}"
+ if not content.strip():
+ # An empty file is treated as no URL excludes by the check, so the key matches the unset case
+ return ""
+ return rat_excludes.content_hash(content)
+
+
async def _resolve_suffixed_file_existence(release: sql.Release, rel_path: str | None = None) -> list[str]:
if (not rel_path) or (not release.latest_revision_number):
return []
@@ -602,6 +620,7 @@ _EXTRA_ARG_RESOLVERS: Final[dict[str, Callable[[sql.Release, str | None], Any]]]
"cross_format_sibling_swhids": _resolve_cross_format_sibling_swhids,
"github_tp_sha": _resolve_github_tp_sha,
"is_podling": _resolve_is_podling,
+ "rat_excludes_hash": _resolve_rat_excludes_hash,
"suffixed_file_existence": _resolve_suffixed_file_existence,
"unsuffixed_file_hash": _resolve_unsuffixed_file_hash,
"unsuffixed_file_uploaders": _resolve_unsuffixed_file_uploaders,
diff --git a/atr/tasks/checks/rat.py b/atr/tasks/checks/rat.py
index e9a438da..1b421c27 100644
--- a/atr/tasks/checks/rat.py
+++ b/atr/tasks/checks/rat.py
@@ -16,6 +16,7 @@
# under the License.
import asyncio
+import dataclasses
import os
import shlex
import subprocess
@@ -25,13 +26,16 @@ from typing import Final
import defusedxml
import defusedxml.ElementTree as ElementTree
+import atr.attestable as attestable
import atr.config as config
import atr.constants as constants
import atr.log as log
+import atr.models.attestable as models_attestable
import atr.models.checkdata as checkdata
import atr.models.results as results
import atr.models.safe as safe
import atr.models.sql as sql
+import atr.rat_excludes as rat_excludes
import atr.sandbox as sandbox
import atr.tasks.checks as checks
import atr.tasks.task as task
@@ -54,6 +58,9 @@ _POLICY_EXCLUDES_FILENAME: Final[str] = ".atr-policy-rat-excludes"
# The name of the file that contains the exclusions for the specified archive
_RAT_EXCLUDES_FILENAME: Final[str] = ".rat-excludes"
+# The name of the temp file for excludes fetched from a project's configured URL
+_URL_EXCLUDES_FILENAME: Final[str] = ".atr-url-rat-excludes"
+
# The name of the RAT report file
_RAT_REPORT_FILENAME: Final[str] = ".atr-rat-report.xml"
@@ -70,14 +77,26 @@ _STD_EXCLUSIONS_EXTENDED: Final[list[str]] = [
]
# Release policy fields which this check relies on - used for result caching
INPUT_POLICY_KEYS: Final[list[str]] = ["license_check_mode", "source_excludes_rat"]
-INPUT_EXTRA_ARGS: Final[list[str]] = []
-CHECK_VERSION: Final[str] = "5"
+# The resolver fetches the configured .rat-excludes URL and returns a hash of its contents, so a
+# change to the fetched file busts the cached result even though the URL string is unchanged
+INPUT_EXTRA_ARGS: Final[list[str]] = ["rat_excludes_hash"]
+CHECK_VERSION: Final[str] = "6"
class RatError(RuntimeError):
pass
+@dataclasses.dataclass
+class _UrlExcludes:
+ # The outcome of fetching a project's configured .rat-excludes URL: content on success, or an
+ # error with transient telling us whether it's worth retrying
+ url: str
+ content: bytes | None = None
+ error: str | None = None
+ transient: bool = False
+
+
async def check(args: checks.FunctionArguments) -> results.Results | None:
"""Use Apache RAT to check the licenses of the files in the artifact."""
recorder = await args.recorder(CHECK_VERSION)
@@ -102,15 +121,35 @@ async def check(args: checks.FunctionArguments) -> results.Results | None:
log.info(f"Checking RAT licenses for {artifact_abs_path} (rel: {args.primary_rel_path})")
policy_excludes = project.policy_source_excludes_rat
+ url_excludes = await _fetch_url_excludes(project)
try:
- await _check_core(args, recorder, archive_dir, policy_excludes)
+ await _check_core(args, recorder, archive_dir, policy_excludes, url_excludes)
except OSError as e:
raise task.CheckRetryableError("Error reading the extracted archive tree", {"error": str(e)}) from e
return None
+async def _fetch_url_excludes(project: sql.Project) -> _UrlExcludes | None:
+ # We fetch here as well as in the cache-key resolver - the resolver's fetch decides whether the
+ # cached result still applies, and this one gets the rules we actually feed to RAT
+ url = project.policy_rat_excludes_url.strip()
+ if not url:
+ return None
+ try:
+ content = await rat_excludes.fetch(url)
+ except rat_excludes.TransientError as e:
+ return _UrlExcludes(url=url, error=str(e), transient=True)
+ except rat_excludes.UnavailableError as e:
+ return _UrlExcludes(url=url, error=str(e), transient=False)
+ if not content.strip():
+ # An empty file more likely means a misconfigured URL than a deliberate no-op, so we
+ # fall through to the policy excludes rather than silently suppressing them
+ return None
+ return _UrlExcludes(url=url, content=content)
+
+
def _build_rat_command(
rat_jar_path: str,
xml_output_path: str,
@@ -163,14 +202,18 @@ async def _check_core(
recorder: checks.Recorder,
archive_dir: safe.StatePath,
policy_excludes: list[str],
+ url_excludes: _UrlExcludes | None,
) -> None:
result = await asyncio.to_thread(
_synchronous,
archive_dir=str(archive_dir),
policy_excludes=policy_excludes,
+ url_excludes=url_excludes,
rat_jar_path=args.extra_args.get("rat_jar_path", _CONFIG.APACHE_RAT_JAR_PATH),
)
+ await _record_url_excludes(args, result, url_excludes)
+
# Record individual file failures before the overall result
for file in result.unknown_license_files:
await recorder.concern("Unknown license", None, member_rel_path=file.name)
@@ -191,6 +234,34 @@ async def _check_core(
await recorder.note(result.message, result_data)
+async def _record_url_excludes(
+ args: checks.FunctionArguments,
+ result: checkdata.Rat,
+ url_excludes: _UrlExcludes | None,
+) -> None:
+ # Stash what we actually fed to RAT from the URL as attestable data for the revision, so a
+ # later analysis can see what was applied. Only when the URL was the effective source -
+ # an archive .rat-excludes wins, and a transient failure gets recorded by the retry instead
+ if (url_excludes is None) or (result.excludes_source != "url") or url_excludes.transient:
+ return
+ if url_excludes.error is not None:
+ record = models_attestable.RatExcludesRecordV2(url=url_excludes.url, error=url_excludes.error)
+ else:
+ content = url_excludes.content or b""
+ record = models_attestable.RatExcludesRecordV2(
+ url=url_excludes.url,
+ content_hash=rat_excludes.content_hash(content),
+ content=content.decode("utf-8", errors="replace"),
+ )
+ await attestable.write_rat_excludes_data(
+ args.project_key,
+ args.version_key,
+ args.revision_number,
+ str(args.primary_rel_path or ""),
+ record,
+ )
+
+
def _check_core_logic_execute_rat(
command: list[str],
scan_root: str,
@@ -342,6 +413,7 @@ def _summary_message(valid: bool, unapproved_licenses: int, unknown_licenses: in
def _synchronous(
archive_dir: str,
policy_excludes: list[str],
+ url_excludes: _UrlExcludes | None = None,
rat_jar_path: str = _CONFIG.APACHE_RAT_JAR_PATH,
) -> checkdata.Rat:
"""Verify license headers using Apache RAT."""
@@ -359,7 +431,7 @@ def _synchronous(
with tempfile.TemporaryDirectory(prefix="rat_scratch_") as scratch_dir:
log.info(f"Created scratch directory: {scratch_dir}")
- return _synchronous_core(archive_dir, scratch_dir, policy_excludes, rat_jar_path)
+ return _synchronous_core(archive_dir, scratch_dir, policy_excludes, url_excludes, rat_jar_path)
def _synchronous_check_jar_exists(rat_jar_path: str) -> tuple[str, checkdata.Rat | None]:
@@ -417,6 +489,7 @@ def _synchronous_core( # noqa: C901
archive_dir: str,
scratch_dir: str,
policy_excludes: list[str],
+ url_excludes: _UrlExcludes | None,
rat_jar_path: str,
) -> checkdata.Rat:
exclude_file_paths: list[str] = []
@@ -438,8 +511,12 @@ def _synchronous_core( # noqa: C901
# Narrow to single path after validation
archive_excludes_path: str | None = exclude_file_paths[0] if exclude_file_paths else None
+ if (error_result := _url_excludes_error_result(archive_excludes_path, url_excludes)) is not None:
+ return error_result
+
+ url_content = url_excludes.content if (url_excludes is not None) else None
excludes_source, effective_excludes_path = _synchronous_core_excludes_source(
- archive_excludes_path, policy_excludes, archive_dir, scratch_dir
+ archive_excludes_path, policy_excludes, url_content, archive_dir, scratch_dir
)
try:
@@ -503,10 +580,31 @@ def _synchronous_core( # noqa: C901
return result
+def _url_excludes_error_result(
+ archive_excludes_path: str | None, url_excludes: _UrlExcludes | None
+) -> checkdata.Rat | None:
+ # An archive .rat-excludes wins, so a URL failure is only a problem without an archive. A
+ # transient failure is non-structural so the check retries, an unavailable one is structural
+ # so it's recorded as a concern
+ if (archive_excludes_path is not None) or (url_excludes is None) or (url_excludes.error is None):
+ return None
+ return checkdata.Rat(
+ message=f"Could not use the configured RAT excludes URL: {url_excludes.error}",
+ errors=[url_excludes.error],
+ excludes_source="url",
+ structural=not url_excludes.transient,
+ )
+
+
def _synchronous_core_excludes_source(
- archive_excludes_path: str | None, policy_excludes: list[str], archive_dir: str, scratch_dir: str
+ archive_excludes_path: str | None,
+ policy_excludes: list[str],
+ url_content: bytes | None,
+ archive_dir: str,
+ scratch_dir: str,
) -> tuple[str, str | None]:
- # Determine excludes_source and effective excludes file
+ # Determine excludes_source and effective excludes file. Precedence is archive, then the
+ # fetched URL, then the hand-typed policy excludes, then defaults only
excludes_source: str
effective_excludes_path: str | None
@@ -514,6 +612,13 @@ def _synchronous_core_excludes_source(
excludes_source = "archive"
effective_excludes_path = os.path.join(archive_dir, archive_excludes_path)
log.info(f"Using archive {_RAT_EXCLUDES_FILENAME}: {archive_excludes_path}")
+ elif url_content is not None:
+ excludes_source = "url"
+ url_excludes_file = os.path.join(scratch_dir, _URL_EXCLUDES_FILENAME)
+ with open(url_excludes_file, "wb") as f:
+ f.write(url_content)
+ effective_excludes_path = url_excludes_file
+ log.info(f"Using excludes fetched from URL, written to: {url_excludes_file}")
elif policy_excludes:
excludes_source = "policy"
policy_excludes_file = os.path.join(scratch_dir, _POLICY_EXCLUDES_FILENAME)
diff --git a/migrations/versions/0125_2026.09.08_fbcb11d5.py b/migrations/versions/0125_2026.09.08_fbcb11d5.py
new file mode 100644
index 00000000..6fba525b
--- /dev/null
+++ b/migrations/versions/0125_2026.09.08_fbcb11d5.py
@@ -0,0 +1,44 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Add a project-configured .rat-excludes URL
+
+Revision ID: 0125_2026.09.08_fbcb11d5
+Revises: 0124_2026.09.02_b7e2f1a9
+Create Date: 2026-09-08 11:13:01.671681+00:00
+"""
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+from alembic import op
+
+# Revision identifiers, used by Alembic
+revision: str = "0125_2026.09.08_fbcb11d5"
+down_revision: str | None = "0124_2026.09.02_b7e2f1a9"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+ with op.batch_alter_table("releasepolicy", schema=None) as batch_op:
+ batch_op.add_column(sa.Column("rat_excludes_url", sa.String(), nullable=False, server_default=""))
+
+
+def downgrade() -> None:
+ with op.batch_alter_table("releasepolicy", schema=None) as batch_op:
+ batch_op.drop_column("rat_excludes_url")
diff --git a/tests/unit/test_asf_yaml_export.py b/tests/unit/test_asf_yaml_export.py
index 0849fd3c..a34e87ad 100644
--- a/tests/unit/test_asf_yaml_export.py
+++ b/tests/unit/test_asf_yaml_export.py
@@ -83,6 +83,7 @@ class _FakePolicy:
"github_vote_workflow_path": [],
"github_finish_workflow_path": [],
"download_path_suffix": "",
+ "rat_excludes_url": "",
}
defaults.update(fields)
for key, value in defaults.items():
@@ -102,6 +103,19 @@ def test_export_includes_download_path_suffix_when_set() -> None:
assert policy["download_path_suffix"] == "{{PROJECT_KEY}}-{{VERSION}}"
+def test_export_includes_rat_excludes_url_when_set() -> None:
+ project = _FakeProject(
+ key="example",
+ committee_key="tooling",
+ name="Apache Example",
+ release_policy=_FakePolicy(rat_excludes_url="https://apache.org/.rat-excludes"),
+ )
+
+ policy = strictyaml.load(projects._asf_yaml_export(project)).data["project"]["policy"]
+
+ assert policy["rat_excludes_url"] == "https://apache.org/.rat-excludes"
+
+
def test_export_includes_only_recipient_keys_that_are_set() -> None:
project = _FakeProject(
key="example",
diff --git a/tests/unit/test_checks_rat.py b/tests/unit/test_checks_rat.py
index cdd709b7..7609fb6f 100644
--- a/tests/unit/test_checks_rat.py
+++ b/tests/unit/test_checks_rat.py
@@ -20,12 +20,14 @@ import shlex
import tarfile
import types
import unittest.mock as mock
+from typing import Any
import pytest
import atr.models.checkdata as checkdata
import atr.models.safe as safe
import atr.models.sql as sql
+import atr.rat_excludes as rat_excludes
import atr.tasks.checks as checks
import atr.tasks.checks.rat as rat
import atr.tasks.task as task
@@ -83,6 +85,7 @@ async def test_check_raises_retryable_for_tooling_errors(tmp_path: pathlib.Path)
project = types.SimpleNamespace(
policy_license_check_mode=sql.LicenseCheckMode.BOTH,
policy_source_excludes_rat=[],
+ policy_rat_excludes_url="",
)
result = checkdata.Rat(
valid=False,
@@ -106,6 +109,7 @@ async def test_check_records_concern_for_structural_errors(tmp_path: pathlib.Pat
project = types.SimpleNamespace(
policy_license_check_mode=sql.LicenseCheckMode.BOTH,
policy_source_excludes_rat=[],
+ policy_rat_excludes_url="",
)
result = checkdata.Rat(
valid=False,
@@ -133,6 +137,7 @@ async def test_check_routes_member_failures_to_concern_and_rollup_to_note(tmp_pa
project = types.SimpleNamespace(
policy_license_check_mode=sql.LicenseCheckMode.BOTH,
policy_source_excludes_rat=[],
+ policy_rat_excludes_url="",
)
result = checkdata.Rat(
valid=False,
@@ -211,6 +216,75 @@ def test_excludes_policy_uses_atr_rat_excludes(rat_available: tuple[bool, bool],
assert rat._RAT_EXCLUDES_FILENAME not in command
+def test_excludes_url_written_and_preferred_over_policy(tmp_path: pathlib.Path):
+ """A fetched URL beats the hand-typed policy excludes and is written to the scratch file."""
+ archive_dir = tmp_path / "archive"
+ scratch_dir = tmp_path / "scratch"
+ archive_dir.mkdir()
+ scratch_dir.mkdir()
+ source, path = rat._synchronous_core_excludes_source(None, ["*.py"], b"*.txt\n", str(archive_dir), str(scratch_dir))
+ assert source == "url"
+ assert path is not None
+ assert pathlib.Path(path).read_bytes() == b"*.txt\n"
+
+
+def test_excludes_archive_preferred_over_url(tmp_path: pathlib.Path):
+ """An archive .rat-excludes wins even when a URL was fetched."""
+ archive_dir = tmp_path / "archive"
+ scratch_dir = tmp_path / "scratch"
+ archive_dir.mkdir()
+ scratch_dir.mkdir()
+ source, _ = rat._synchronous_core_excludes_source(
+ ".rat-excludes", ["*.py"], b"*.txt\n", str(archive_dir), str(scratch_dir)
+ )
+ assert source == "archive"
+
+
+@pytest.mark.asyncio
+async def test_fetch_url_excludes_none_when_unset() -> None:
+ project: Any = types.SimpleNamespace(policy_rat_excludes_url="")
+ assert await rat._fetch_url_excludes(project) is None
+
+
+@pytest.mark.asyncio
+async def test_fetch_url_excludes_carries_content_on_success() -> None:
+ project: Any = types.SimpleNamespace(policy_rat_excludes_url="https://apache.org/.rat-excludes")
+ with mock.patch.object(rat_excludes, "fetch", new=mock.AsyncMock(return_value=b"*.txt\n")):
+ result = await rat._fetch_url_excludes(project)
+ assert result is not None
+ assert result.content == b"*.txt\n"
+ assert result.error is None
+ assert result.transient is False
+
+
+@pytest.mark.asyncio
+async def test_fetch_url_excludes_none_when_body_empty() -> None:
+ # An empty fetched file falls through to policy excludes rather than suppressing them
+ project: Any = types.SimpleNamespace(policy_rat_excludes_url="https://apache.org/.rat-excludes")
+ with mock.patch.object(rat_excludes, "fetch", new=mock.AsyncMock(return_value=b" \n")):
+ assert await rat._fetch_url_excludes(project) is None
+
+
+@pytest.mark.asyncio
+async def test_fetch_url_excludes_marks_transient_failure_for_retry() -> None:
+ project: Any = types.SimpleNamespace(policy_rat_excludes_url="https://apache.org/.rat-excludes")
+ with mock.patch.object(rat_excludes, "fetch", new=mock.AsyncMock(side_effect=rat_excludes.TransientError("down"))):
+ result = await rat._fetch_url_excludes(project)
+ assert result is not None
+ assert result.transient is True
+ assert result.content is None
+
+
+@pytest.mark.asyncio
+async def test_fetch_url_excludes_records_unavailable_failure() -> None:
+ project: Any = types.SimpleNamespace(policy_rat_excludes_url="https://apache.org/.rat-excludes")
+ with mock.patch.object(rat_excludes, "fetch", new=mock.AsyncMock(side_effect=rat_excludes.UnavailableError("404"))):
+ result = await rat._fetch_url_excludes(project)
+ assert result is not None
+ assert result.transient is False
+ assert result.error == "404"
+
+
def test_sanitise_command_replaces_absolute_paths():
command = [
"java",
diff --git a/tests/unit/test_rat_excludes.py b/tests/unit/test_rat_excludes.py
new file mode 100644
index 00000000..ea27fee5
--- /dev/null
+++ b/tests/unit/test_rat_excludes.py
@@ -0,0 +1,87 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import unittest.mock as mock
+
+import pytest
+
+import atr.models.validation as validation
+import atr.rat_excludes as rat_excludes
+import atr.util as util
+
+
+def test_empty_url_is_allowed() -> None:
+ # An unset URL means we don't fetch anything, so it must pass validation
+ validation.validate_rat_excludes_url("")
+ validation.validate_rat_excludes_url(" ")
+
+
+@pytest.mark.parametrize(
+ "url",
+ [
+ "https://gitbox.apache.org/repos/asf/example.git/.rat-excludes",
+ "https://apache.org/.rat-excludes",
+ "https://raw.githubusercontent.com/apache/example/main/.rat-excludes",
+ ],
+)
+def test_allowed_hosts_pass(url: str) -> None:
+ validation.validate_rat_excludes_url(url)
+
+
+@pytest.mark.parametrize(
+ "url",
+ [
+ "http://gitbox.apache.org/.rat-excludes",
+ "https://example.com/.rat-excludes",
+ "https://apache.org.evil.com/.rat-excludes",
+ "ftp://apache.org/.rat-excludes",
+ "https:///.rat-excludes",
+ ],
+)
+def test_disallowed_urls_raise(url: str) -> None:
+ with pytest.raises(ValueError):
+ validation.validate_rat_excludes_url(url)
+
+
+def _mock_session(response: mock.Mock) -> mock.MagicMock:
+ session = mock.Mock()
+ session.get = mock.AsyncMock(return_value=response)
+ session_cm = mock.MagicMock()
+ session_cm.__aenter__ = mock.AsyncMock(return_value=session)
+ session_cm.__aexit__ = mock.AsyncMock(return_value=False)
+ return session_cm
+
+
+@pytest.mark.asyncio
+async def test_redirect_is_treated_as_unavailable() -> None:
+ # An allowlisted host that redirects elsewhere must not be followed
+ response = mock.Mock(status=302)
+ response.raise_for_status = mock.Mock()
+ response.content = mock.Mock(read=mock.AsyncMock(return_value=b""))
+ with mock.patch.object(util, "create_secure_session", return_value=_mock_session(response)):
+ with pytest.raises(rat_excludes.UnavailableError):
+ await rat_excludes.fetch("https://apache.org/.rat-excludes")
+
+
+@pytest.mark.asyncio
+async def test_oversized_body_is_unavailable() -> None:
+ response = mock.Mock(status=200)
+ response.raise_for_status = mock.Mock()
+ response.content = mock.Mock(read=mock.AsyncMock(return_value=b"x" * (64 * 1024 + 1)))
+ with mock.patch.object(util, "create_secure_session", return_value=_mock_session(response)):
+ with pytest.raises(rat_excludes.UnavailableError):
+ await rat_excludes.fetch("https://apache.org/.rat-excludes")
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org