You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tooling.apache.org by ar...@apache.org on 2026/06/24 15:47:11 UTC
(tooling-trusted-releases) branch arm updated (88a4176a -> 6d1eac51)
This is an automated email from the ASF dual-hosted git repository.
alitheg pushed a change to branch arm
in repository https://gitbox.apache.org/repos/asf/tooling-trusted-releases.git
discard 88a4176a #478, #479 - fix some path issues with certain PMCs
add 76a745aa Limit how many nested files can be published in a release
add 5fb124e8 Make navigation elements on revision pages consistent
add 87d228e1 Remove all remaining revision features from the finish phase
add 770c39ee Fix a race condition in project deletion
new 6d1eac51 #478, #479 - fix some path issues with certain PMCs
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (88a4176a)
\
N -- N -- N refs/heads/arm (6d1eac51)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
atr/get/finish.py | 188 +---------------------------
atr/get/revisions.py | 44 +------
atr/post/finish.py | 87 +------------
atr/post/revisions.py | 16 +--
atr/shared/finish.py | 19 +--
atr/storage/writers/project.py | 106 +++++++++-------
atr/storage/writers/release.py | 132 -------------------
atr/tasks/checks/paths.py | 33 ++++-
tests/unit/test_documentation_tree_check.py | 63 ++++++++++
tests/unit/test_project_writer_delete.py | 123 ++++++++++++++++++
10 files changed, 291 insertions(+), 520 deletions(-)
create mode 100644 tests/unit/test_documentation_tree_check.py
create mode 100644 tests/unit/test_project_writer_delete.py
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org
(tooling-trusted-releases) 01/01: #478, #479 - fix some path issues with certain PMCs
Posted by ar...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
alitheg pushed a commit to branch arm
in repository https://gitbox.apache.org/repos/asf/tooling-trusted-releases.git
commit 6d1eac51f70b274c91bdf32af2f34d48fb0a8c46
Author: Alastair McFarlane <ar...@apache.org>
AuthorDate: Wed Jun 24 16:38:58 2026 +0100
#478, #479 - fix some path issues with certain PMCs
---
atr/classify.py | 2 +-
atr/svn/catalog.py | 167 ++---------------------------
atr/svn/dist.py | 248 +++++++++++++++++++++++++++++++++++++++++++
tests/unit/test_classify.py | 5 +
tests/unit/test_decompose.py | 156 +++++++++++++++++++++++++++
5 files changed, 418 insertions(+), 160 deletions(-)
diff --git a/atr/classify.py b/atr/classify.py
index d253e84e..93775996 100644
--- a/atr/classify.py
+++ b/atr/classify.py
@@ -110,7 +110,7 @@ _SRC_SOURCE_RE: Final[re.Pattern[str]] = re.compile(r"(^|[-_.])(project|source|s
_TOKEN_SPLIT_RE: Final[re.Pattern[str]] = re.compile(r"[-_.]+")
-_DEFAULT_BINARY_EXTENSIONS: Final[frozenset[str]] = frozenset({".jar"})
+_DEFAULT_BINARY_EXTENSIONS: Final[frozenset[str]] = frozenset({".jar", ".whl"})
class FileType(enum.Enum):
diff --git a/atr/svn/catalog.py b/atr/svn/catalog.py
index b8441b65..969fe11f 100644
--- a/atr/svn/catalog.py
+++ b/atr/svn/catalog.py
@@ -16,13 +16,12 @@
# under the License.
"""
-Decompose dist paths into project and version, and catalogue dist commits.
+Catalogue published dist commits into the database via the storage interface.
"""
import dataclasses
import datetime
import pathlib
-import re
from typing import Final
import atr.analysis as analysis
@@ -35,66 +34,22 @@ import atr.models.sql as sql
import atr.storage as storage
import atr.storage.writers.release as release
import atr.svn as svn
+import atr.svn.dist as dist
# Changed paths are relative to the dist repo root
# We only watch published releases; dev candidate activity is out of scope
_RELEASE_PREFIX: Final[str] = "release/"
-# Verified dist-path -> projects.json key remaps; subproject is None for a committee-level layout
-# Per-entry only - a blanket rule gives false positives (camel-karaf is Camel's, not Karaf)
-DIST_PROJECT_REMAPS: Final[dict[tuple[str, str | None], str]] = {
- # Artemis graduated from ActiveMQ; dist still splits it across both areas
- ("activemq", "activemq-artemis"): "artemis",
- # httpd's top level is the HTTP Server, and the committee has two projects.json
- # entries so the bare-name rule can't pick
- ("httpd", None): "httpd-http_server",
- # Dist names that differ too much from the projects.json key for the bare-name match
- ("trafficcontrol", None): "attic-traffic-control",
- ("xmlgraphics", "commons"): "xmlgraphics-xml-graphics-commons",
-}
-
-# Wheels ship beside the source tarball (airflow providers), so they're a build, not a release
-IGNORED_ARTIFACT_SUFFIXES: Final[tuple[str, ...]] = (".whl",)
-
# Companion files paired to an artifact by basename, strongest first
_SIGNATURE_SUFFIXES: Final[tuple[str, ...]] = (".asc", ".sig")
_CHECKSUM_SUFFIXES: Final[tuple[str, ...]] = (".sha512", ".sha256", ".sha1", ".sha", ".mds", ".md5")
_SBOM_SUFFIXES: Final[tuple[str, ...]] = (".cdx.json", ".cdx.xml")
-# A version-ish component; anything it misses falls through to filename parsing
-_VERSION_RE: Final[re.Pattern[str]] = re.compile(r"^v?\d+(?:\.\d+)*(?:[.\-_][A-Za-z0-9]+)*$")
-
-# A bare single digit is a major-version grouping dir (xerces/c/2), not a version
-_BARE_DIGIT_RE: Final[re.Pattern[str]] = re.compile(r"^\d$")
-
-# Splits a flat name-version; non-greedy, so spark-4.1.0-preview1 comes apart at the version
-_COMBINED_RE: Final[re.Pattern[str]] = re.compile(r"^(?P<name>.+?)-(?P<version>v?\d+(?:\.\d+)*(?:[.\-][A-Za-z0-9]+)*)$")
-
-# Semver in a filename: dotted numerics plus an ASF qualifier tail. The lookbehind blocks
-# only a leading digit, so a dotted classifier (Xerces-J-bin.2.10.0) still parses; underscore
-# isn't a separator
-_SEMVER_FILE_RE: Final[re.Pattern[str]] = re.compile(
- r"(?<![0-9])(\d+(?:\.\d+)+(?:[.\-](?:alpha|beta|rc|cr|m|milestone|preview|pre|incubating|final|ga)[0-9]*)*)",
- re.IGNORECASE,
-)
-
-# Calver dates (2026-01-27 or 20050330); tried only when there's no semver, and
-# limited to 19xx/20xx so we don't grab an arbitrary run of digits
-_CALVER_FILE_RE: Final[re.Pattern[str]] = re.compile(r"(?<!\d)(\d{4}-\d{2}-\d{2}|(?:19|20)\d{6})(?!\d)")
-
# A release resolved against ATR's projects, ready to hand to the system actor
type _ResolvedRelease = tuple[safe.ProjectKey, safe.VersionKey, list[release.ArtifactInput]]
type _ResolvedArchive = tuple[safe.ProjectKey, safe.VersionKey]
-@dataclasses.dataclass(frozen=True)
-class Decomposed:
- subproject: str | None
- version: str | None
- # "dir", "combined", "filename" or "unknown"
- source: str
-
-
@dataclasses.dataclass
class _ReleaseFiles:
# Added files for one release in a commit as (dir-under-release, basename, classification),
@@ -106,18 +61,6 @@ class _ReleaseFiles:
has_source: bool = False
-def candidate_keys(committee: str, subproject: str | None) -> list[str]:
- # Dist layout and projects.json keys don't always line up (commons/lang -> commons-lang, but
- # accumulo/accumulo-access is already its key), so try the obvious shapes; apache- is stripped
- if not subproject:
- return [committee]
- bare = subproject.removeprefix("apache-")
- keys: list[str] = []
- for name in dict.fromkeys([bare, subproject]):
- keys += [name, f"{committee}-{name}"]
- return keys
-
-
async def catalogue_commit(commit: dict) -> None:
if str(commit.get("committer", "")) == svn.ASF_TOOL:
# Our own publishes are already in the database
@@ -143,47 +86,6 @@ async def catalogue_commit(commit: dict) -> None:
await _apply_changes(system, releases, archives, date)
-def decompose(committee: str, parts: tuple[str, ...], filename: str | None = None) -> Decomposed | None:
- # parts are the dirs below the committee, filename the release file if any. The filename
- # wins for the version (a .../4.5/ series dir covers 4.5.x; the file pins 4.5.13), dir is the fallback
- parts = _strip_self_prefix(committee, parts)
- if not parts:
- return _filename_only(committee, filename)
-
- subproject, dir_version, dir_source = _subproject_and_dir_version(committee, parts)
- name_version = version_from_filename(filename) if filename else None
- if name_version is not None:
- return Decomposed(subproject=subproject, version=name_version, source="filename")
- return Decomposed(subproject=subproject, version=dir_version, source=dir_source)
-
-
-def missing_project_key(committee: str, subproject: str | None) -> str:
- # An ATR-style key for a project projects.json lacks, so releases can FK to it
- # apache- stripped, and we avoid doubling the committee prefix
- if not subproject:
- return committee
- bare = subproject.removeprefix("apache-")
- if bare.startswith(f"{committee}-"):
- return bare
- return f"{committee}-{bare}"
-
-
-def normalise(name: str) -> str:
- # So traffic_server, traffic-server and "Traffic Server" all compare equal
- return re.sub(r"[^a-z0-9]", "", name.lower())
-
-
-def version_from_filename(filename: str) -> str | None:
- # Semver wins over calver so a dotted release isn't misread as a date
- semver = _SEMVER_FILE_RE.search(filename)
- if semver is not None:
- return semver.group(1)
- calver = _CALVER_FILE_RE.search(filename)
- if calver is not None:
- return calver.group(1)
- return None
-
-
async def _apply_changes(
system: release.FoundationAdmin,
releases: list[_ResolvedRelease],
@@ -211,7 +113,7 @@ def _artifacts(rel_files: _ReleaseFiles) -> list[release.ArtifactInput]:
for dirpath, name, file_type in rel_files.files:
if (name in seen) or (file_type in (classify.FileType.METADATA, classify.FileType.DISALLOWED)):
continue
- if (not analysis.is_artifact(name)) or name.endswith(IGNORED_ARTIFACT_SUFFIXES):
+ if (not analysis.is_artifact(name)) or name.endswith(dist.IGNORED_ARTIFACT_SUFFIXES):
continue
seen.add(name)
siblings = by_dir[dirpath]
@@ -244,7 +146,7 @@ def _companion(siblings: set[str], artifact: str, suffixes: tuple[str, ...]) ->
return None
-def _decompose_change(path: str) -> tuple[str, Decomposed, bool, str | None] | None:
+def _decompose_change(path: str) -> tuple[str, dist.Decomposed, bool, str | None] | None:
# release/<committee>/... only; returns committee, decomposition, is_dir and
# the filename (None for a directory)
if not path.startswith(_RELEASE_PREFIX):
@@ -259,36 +161,16 @@ def _decompose_change(path: str) -> tuple[str, Decomposed, bool, str | None] | N
if not rest:
return None
if is_dir:
- decomposed = decompose(committee, tuple(rest), None)
+ decomposed = dist.decompose(committee, tuple(rest), None)
filename = None
else:
filename = rest[-1]
- decomposed = decompose(committee, tuple(rest[:-1]), filename)
+ decomposed = dist.decompose(committee, tuple(rest[:-1]), filename)
if decomposed is None:
return None
return committee, decomposed, is_dir, filename
-def _filename_only(committee: str, filename: str | None) -> Decomposed | None:
- # A file directly under the committee has no dir to name the subproject, so it comes from
- # the filename before the version (sling does this); a name == committee is a TLP release
- if not filename:
- return None
- version = version_from_filename(filename)
- if version is None:
- return None
- name = filename[: filename.find(version)].rstrip("-._")
- if (not name) or (name.removeprefix("apache-") == committee):
- return Decomposed(subproject=None, version=version, source="filename")
- return Decomposed(subproject=name, version=version, source="filename")
-
-
-def _is_version_dir(component: str) -> bool:
- if _BARE_DIGIT_RE.match(component):
- return False
- return bool(_VERSION_RE.match(component))
-
-
def _report(
releases: list[_ResolvedRelease],
archives: list[_ResolvedArchive],
@@ -354,8 +236,8 @@ async def _resolve_release(data: db.Session, rel_files: _ReleaseFiles) -> _Resol
async def _resolve_project(data: db.Session, committee: str, subproject: str | None) -> sql.Project | None:
# Resolve against what ATR already has, via the same remaps and candidate keys
# as the backfill. The watcher never invents projects; unknowns are logged
- remapped = DIST_PROJECT_REMAPS.get((committee, subproject))
- candidates = [remapped] if remapped is not None else candidate_keys(committee, subproject)
+ remapped = dist.PROJECT_REMAPS.get((committee, subproject))
+ candidates = [remapped] if remapped is not None else dist.candidate_keys(committee, subproject)
for candidate in candidates:
project = await data.project(key=candidate).get()
if project is not None:
@@ -371,21 +253,6 @@ def _safe_keys(project_key: str, version: str) -> tuple[safe.ProjectKey, safe.Ve
return None
-def _split_combined(component: str) -> tuple[str, str] | None:
- match = _COMBINED_RE.match(component)
- if match is None:
- return None
- return match.group("name"), match.group("version")
-
-
-def _strip_self_prefix(committee: str, parts: tuple[str, ...]) -> tuple[str, ...]:
- # jackrabbit/jackrabbit/... and ace/apache-ace/... are the TLP's own area, not
- # a subproject. Drop a leading dir that just repeats the committee
- if parts and (parts[0] in (committee, f"apache-{committee}")):
- return parts[1:]
- return parts
-
-
def _structural_changes(
changed: dict,
) -> tuple[dict[tuple[str, str | None, str], _ReleaseFiles], set[tuple[str, str | None, str]]]:
@@ -418,21 +285,3 @@ def _structural_changes(
if file_type is classify.FileType.SOURCE:
bundle.has_source = True
return {key: bundle for key, bundle in added.items() if bundle.has_source}, removed
-
-
-def _subproject_and_dir_version(committee: str, parts: tuple[str, ...]) -> tuple[str | None, str | None, str]:
- # First dir decides the subproject; the version is whichever dir looks like one
- first = parts[0]
- if _is_version_dir(first):
- return None, first, "dir"
- combined = _split_combined(first)
- if combined is not None:
- name, version = combined
- return (None if name == committee else name), version, "combined"
- for part in parts[1:]:
- if _is_version_dir(part):
- return first, part, "dir"
- deeper = _split_combined(part)
- if deeper is not None:
- return first, deeper[1], "combined"
- return first, None, "unknown"
diff --git a/atr/svn/dist.py b/atr/svn/dist.py
new file mode 100644
index 00000000..96e63ddf
--- /dev/null
+++ b/atr/svn/dist.py
@@ -0,0 +1,248 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+Decompose dist.apache.org paths into a project subproject and version.
+"""
+
+import dataclasses
+import re
+from typing import Final, Literal
+
+# Verified dist-path -> projects.json key remaps; subproject is None for a committee-level layout
+# Per-entry only - a blanket rule gives false positives (camel-karaf is Camel's, not Karaf)
+PROJECT_REMAPS: Final[dict[tuple[str, str | None], str]] = {
+ # Artemis graduated from ActiveMQ; dist still splits it across both areas
+ ("activemq", "activemq-artemis"): "artemis",
+ # httpd's top level is the HTTP Server, and the committee has two projects.json
+ # entries so the bare-name rule can't pick
+ ("httpd", None): "httpd-http_server",
+ # Dist names that differ too much from the projects.json key for the bare-name match
+ ("trafficcontrol", None): "attic-traffic-control",
+ ("xmlgraphics", "commons"): "xmlgraphics-xml-graphics-commons",
+}
+
+# Wheels ship beside the source tarball (airflow providers), so they're a build, not a release
+IGNORED_ARTIFACT_SUFFIXES: Final[tuple[str, ...]] = (".whl",)
+
+# Dirs that hold a project's sub-packages or builds rather than naming a subproject, so the
+# subproject and version live in the filename (airflow's providers/, a source/ or binaries/
+# build split, a releases/ container). Matched case-insensitively against the first dir
+_GROUPING_BUCKETS: Final[frozenset[str]] = frozenset(
+ {"providers", "source", "sources", "binaries", "bin", "src", "releases"}
+)
+
+# Buckets that only apply under a specific committee, where the name is a real subproject
+# elsewhere (maven/plugins holds one release per plugin, but plugins isn't always a bucket)
+_COMMITTEE_BUCKETS: Final[frozenset[tuple[str, str]]] = frozenset({("maven", "plugins")})
+
+# Build/status tokens that leak onto a filename's name (chukwa-src, chukwa-incubating-src,
+# a stray v from a v-prefixed version); stripped from the subproject only, never the filename
+_NAME_BUILD_SUFFIXES: Final[frozenset[str]] = frozenset(
+ {"src", "source", "sources", "bin", "binaries", "incubating", "v"}
+)
+
+# A version-ish component; anything it misses falls through to filename parsing
+_VERSION_RE: Final[re.Pattern[str]] = re.compile(r"^v?\d+(?:\.\d+)*(?:[.\-_][A-Za-z0-9]+)*$")
+
+# A bare single digit is a major-version grouping dir (xerces/c/2), not a version
+_BARE_DIGIT_RE: Final[re.Pattern[str]] = re.compile(r"^\d$")
+
+# Splits a flat name-version; non-greedy, so spark-4.1.0-preview1 comes apart at the version
+_COMBINED_RE: Final[re.Pattern[str]] = re.compile(r"^(?P<name>.+?)-(?P<version>v?\d+(?:\.\d+)*(?:[.\-][A-Za-z0-9]+)*)$")
+
+# Semver in a filename: dotted numerics plus an ASF qualifier tail. The lookbehind blocks
+# only a leading digit, so a dotted classifier (Xerces-J-bin.2.10.0) still parses; underscore
+# isn't a separator
+_SEMVER_FILE_RE: Final[re.Pattern[str]] = re.compile(
+ r"(?<![0-9])(\d+(?:\.\d+)+(?:[.\-](?:alpha|beta|rc|cr|m|milestone|preview|pre|incubating|final|ga)[0-9]*)*)",
+ re.IGNORECASE,
+)
+
+# Calver dates (2026-01-27 or 20050330); tried only when there's no semver, and
+# limited to 19xx/20xx so we don't grab an arbitrary run of digits
+_CALVER_FILE_RE: Final[re.Pattern[str]] = re.compile(r"(?<!\d)(\d{4}-\d{2}-\d{2}|(?:19|20)\d{6})(?!\d)")
+
+
+@dataclasses.dataclass(frozen=True)
+class Decomposed:
+ subproject: str | None
+ version: str | None
+ source: Literal["dir", "combined", "filename", "unknown"]
+
+
+def candidate_keys(committee: str, subproject: str | None) -> list[str]:
+ # Dist layout and projects.json keys don't always line up (commons/lang -> commons-lang, but
+ # accumulo/accumulo-access is already its key), so try the obvious shapes; apache- is stripped
+ if not subproject:
+ return [committee]
+ bare = subproject.removeprefix("apache-")
+ keys: list[str] = []
+ for name in dict.fromkeys([bare, subproject]):
+ keys += [name, f"{committee}-{name}"]
+ return keys
+
+
+def decompose(committee: str, parts: tuple[str, ...], filename: str | None = None) -> Decomposed | None:
+ # parts are the dirs below the committee, filename the release file if any. A grouping bucket
+ # aside, the subproject comes from the first dir and the version from the filename when it has
+ # one (a .../4.5/ series dir covers 4.5.x; the file pins 4.5.13), with the dir as the fallback
+ parts = _strip_self_prefix(committee, parts)
+ if not parts:
+ return _filename_only(committee, filename)
+ if _is_grouping_bucket(committee, parts, filename):
+ # The dirs are just a distribution bucket (airflow's providers/<series>), so the
+ # subproject and version both live in the filename, as in the flat layout
+ return _filename_only(committee, filename)
+ subproject, dir_version, dir_source = _subproject_and_dir_version(committee, parts)
+ version, source = _choose_version(dir_version, dir_source, filename)
+ return Decomposed(subproject=subproject, version=version, source=source)
+
+
+def missing_project_key(committee: str, subproject: str | None) -> str:
+ # An ATR-style key for a project projects.json lacks, so releases can FK to it
+ # apache- stripped, and we avoid doubling the committee prefix
+ if not subproject:
+ return committee
+ bare = subproject.removeprefix("apache-")
+ if bare.startswith(f"{committee}-"):
+ return bare
+ return f"{committee}-{bare}"
+
+
+def normalise(name: str) -> str:
+ # So traffic_server, traffic-server and "Traffic Server" all compare equal
+ return re.sub(r"[^a-z0-9]", "", name.lower())
+
+
+def version_from_filename(filename: str) -> str | None:
+ # Semver wins over calver so a dotted release isn't misread as a date
+ semver = _SEMVER_FILE_RE.search(filename)
+ if semver is not None:
+ return semver.group(1)
+ calver = _CALVER_FILE_RE.search(filename)
+ if calver is not None:
+ return calver.group(1)
+ return None
+
+
+def _choose_version(
+ dir_version: str | None, dir_source: Literal["dir", "combined", "unknown"], filename: str | None
+) -> tuple[str | None, Literal["dir", "combined", "filename", "unknown"]]:
+ # The dir and the filename can each carry a version. The filename usually wins (a 4.5/ series
+ # dir vs the 4.5.13 file), but when its leading version is a distractor (kafka_2.13-4.3.1.tgz,
+ # the 2.13 Scala build) and the dir pins a version the filename confirms, the dir wins - unless
+ # the filename merely refines the dir version (a 1.0.0-incubating tail)
+ if dir_version is not None:
+ dir_version = _normalise_version(dir_version)
+ name_version = version_from_filename(filename) if filename else None
+ if name_version is None:
+ return dir_version, dir_source
+ dir_confirmed = (dir_version is not None) and (filename is not None) and _version_in_filename(dir_version, filename)
+ if dir_confirmed and (dir_version is not None) and not name_version.startswith(dir_version):
+ return dir_version, dir_source
+ return name_version, "filename"
+
+
+def _filename_only(committee: str, filename: str | None) -> Decomposed | None:
+ # A file directly under the committee has no dir to name the subproject, so it comes from
+ # the filename before the version (sling does this); a name == committee is a TLP release
+ if not filename:
+ return None
+ version = version_from_filename(filename)
+ if version is None:
+ return None
+ name = _strip_name_suffixes(filename[: filename.find(version)].rstrip("-._"))
+ if (not name) or (name.removeprefix("apache-") == committee):
+ return Decomposed(subproject=None, version=version, source="filename")
+ return Decomposed(subproject=name, version=version, source="filename")
+
+
+def _is_grouping_bucket(committee: str, parts: tuple[str, ...], filename: str | None) -> bool:
+ # A leading dir that's a bucket holds a project's sub-packages or builds rather than
+ # naming a subproject, so the subproject and version come from the filename. The
+ # subproject may be null (a flat TLP release like myfaces/source/myfaces-1.0.9-src.tgz)
+ # or named (airflow/providers/apache_airflow_providers_<name>); either way the filename
+ # must carry a version, or there's nothing for the bucket to key on
+ if filename is None:
+ return False
+ lead = parts[0].lower()
+ if (lead not in _GROUPING_BUCKETS) and ((committee, lead) not in _COMMITTEE_BUCKETS):
+ return False
+ return _filename_only(committee, filename) is not None
+
+
+def _is_version_dir(component: str) -> bool:
+ if _BARE_DIGIT_RE.match(component):
+ return False
+ return bool(_VERSION_RE.match(component))
+
+
+def _normalise_version(version: str) -> str:
+ # A version dir sometimes uses underscores as the separator (0_92, 1_4_7); the release
+ # key uses dots. Only the key changes - the real path keeps its underscores for downloads
+ return version.replace("_", ".")
+
+
+def _split_combined(component: str) -> tuple[str, str] | None:
+ match = _COMBINED_RE.match(component)
+ if match is None:
+ return None
+ return match.group("name"), match.group("version")
+
+
+def _strip_name_suffixes(name: str) -> str:
+ # Drop trailing build/status tokens (chukwa-incubating-src -> chukwa); keep at least one
+ # token so a name that is only a suffix word survives
+ tokens = name.split("-")
+ while (len(tokens) > 1) and (tokens[-1].lower() in _NAME_BUILD_SUFFIXES):
+ tokens.pop()
+ return "-".join(tokens)
+
+
+def _strip_self_prefix(committee: str, parts: tuple[str, ...]) -> tuple[str, ...]:
+ # jackrabbit/jackrabbit/... and ace/apache-ace/... are the TLP's own area, not
+ # a subproject. Drop a leading dir that just repeats the committee
+ if parts and (parts[0] in (committee, f"apache-{committee}")):
+ return parts[1:]
+ return parts
+
+
+def _subproject_and_dir_version(
+ committee: str, parts: tuple[str, ...]
+) -> tuple[str | None, str | None, Literal["dir", "combined", "unknown"]]:
+ # First dir decides the subproject; the version is whichever dir looks like one
+ first = parts[0]
+ if _is_version_dir(first):
+ return None, first, "dir"
+ combined = _split_combined(first)
+ if combined is not None:
+ name, version = combined
+ return (None if name == committee else name), version, "combined"
+ for part in parts[1:]:
+ if _is_version_dir(part):
+ return first, part, "dir"
+ deeper = _split_combined(part)
+ if deeper is not None:
+ return first, deeper[1], "combined"
+ return first, None, "unknown"
+
+
+def _version_in_filename(version: str, filename: str) -> bool:
+ # The version appears in the filename as its own token, not as the prefix of a longer
+ # version (4.5 inside 4.5.13) nor tacked onto another number
+ return re.search(rf"(?<![\d.]){re.escape(version)}(?!\.?\d)", filename) is not None
diff --git a/tests/unit/test_classify.py b/tests/unit/test_classify.py
index 6b08a99b..6e13b471 100644
--- a/tests/unit/test_classify.py
+++ b/tests/unit/test_classify.py
@@ -99,6 +99,11 @@ async def test_jar_defaults_to_binary():
assert (await classify.classify(path)) == classify.FileType.BINARY
+async def test_whl_defaults_to_binary():
+ path = safe.RelPath("apache_airflow_providers_airbyte-5.0.2-py3-none-any.whl")
+ assert (await classify.classify(path)) == classify.FileType.BINARY
+
+
def test_matchers_from_policy_builds_both(tmp_path):
source_matcher, binary_matcher = classify.matchers_from_policy(["*-src.*"], ["*-bin.*"], safe.StatePath(tmp_path))
assert source_matcher is not None
diff --git a/tests/unit/test_decompose.py b/tests/unit/test_decompose.py
new file mode 100644
index 00000000..b7f92d5e
--- /dev/null
+++ b/tests/unit/test_decompose.py
@@ -0,0 +1,156 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import pathlib
+
+import pytest
+
+import atr.classify as classify
+import atr.svn.dist as dist
+
+_SOURCE = classify.FileType.SOURCE
+_BINARY = classify.FileType.BINARY
+
+# Each case is a real dist layout we've had to handle, as (committee, dirs-below-committee,
+# filename, expected subproject, expected version, expected classification).
+_CASES: list[tuple[str, tuple[str, ...], str | None, str | None, str | None, classify.FileType | None]] = [
+ # Airflow providers: providers/ is a distribution bucket, so each provider's
+ # identity and version come from the filename, not the dir
+ (
+ "airflow",
+ ("providers",),
+ "apache_airflow_providers_alibaba-3.3.2.tar.gz",
+ "apache_airflow_providers_alibaba",
+ "3.3.2",
+ _SOURCE,
+ ),
+ # The same provider under an airflow-version series dir still keys on the filename
+ (
+ "airflow",
+ ("providers", "2.11"),
+ "apache_airflow_providers_fab-1.5.4.tar.gz",
+ "apache_airflow_providers_fab",
+ "1.5.4",
+ _SOURCE,
+ ),
+ # A wheel rides beside the source for the same provider and is a binary build
+ (
+ "airflow",
+ ("providers",),
+ "apache_airflow_providers_alibaba-3.3.2-py3-none-any.whl",
+ "apache_airflow_providers_alibaba",
+ "3.3.2",
+ _BINARY,
+ ),
+ # The providers bundle is calver and has no per-provider suffix
+ (
+ "airflow",
+ ("providers",),
+ "apache_airflow_providers-2025-11-03-source.tar.gz",
+ "apache_airflow_providers",
+ "2025-11-03",
+ _SOURCE,
+ ),
+ # The older flat layout (file directly under the committee) already worked
+ (
+ "airflow",
+ (),
+ "apache_airflow_providers_airbyte-5.0.2.tar.gz",
+ "apache_airflow_providers_airbyte",
+ "5.0.2",
+ _SOURCE,
+ ),
+ (
+ "airflow",
+ (),
+ "apache_airflow_providers_airbyte-5.0.2-py3-none-any.whl",
+ "apache_airflow_providers_airbyte",
+ "5.0.2",
+ _BINARY,
+ ),
+ # The airflow core itself is a TLP release: the name is just the committee
+ ("airflow", (), "apache-airflow-2.5.0.tar.gz", None, "2.5.0", _SOURCE),
+ # AGE keeps its version in the dir and the filename name is the committee, so it
+ # stays a TLP release - the dir is not echoed in the filename, so not a bucket
+ ("age", ("1.0.0",), "apache-age-1.0.0-incubating-src.tar.gz", None, "1.0.0-incubating", _SOURCE),
+ # Sling encodes the subproject in a flat filename under the committee
+ ("sling", (), "adapter-annotations-1.0.0-source-release.zip", "adapter-annotations", "1.0.0", _SOURCE),
+ # Commons names the subproject in the dir; source/ and binaries/ only split the build,
+ # and the subproject resolves to commons-attributes later via missing_project_key
+ ("commons", ("attributes", "source"), "commons-attributes-2.2-src.tar.gz", "attributes", "2.2", _SOURCE),
+ ("commons", ("attributes", "binaries"), "commons-attributes-2.2.tar.gz", "attributes", "2.2", _BINARY),
+ # A self-prefixed area (jackrabbit/jackrabbit) is the TLP's own, not a subproject
+ ("jackrabbit", ("jackrabbit", "2.20.1"), "jackrabbit-2.20.1.tar.gz", None, "2.20.1", _SOURCE),
+ # A combined name-version dir splits into subproject and version
+ ("spark", ("spark-kubernetes-operator-0.8.0",), None, "spark-kubernetes-operator", "0.8.0", None),
+ # A qualifier tail stays part of the version
+ ("spark", (), "spark-4.1.0-preview1.tar.gz", None, "4.1.0-preview1", _SOURCE),
+ # Kafka's binary carries the Scala version as a distractor; the dir pins the real one,
+ # so the binary and the source below land on the same release. The plain .tgz binary
+ # has no marker, so it falls to source, but both still attach to kafka 4.3.1
+ ("kafka", ("4.3.1",), "kafka_2.13-4.3.1.tgz", None, "4.3.1", _SOURCE),
+ ("kafka", ("4.3.1",), "kafka-4.3.1-src.tgz", None, "4.3.1", _SOURCE),
+ # A series dir is still less specific than the filename, which pins the point release
+ ("example", ("4.5",), "example-4.5.13.tar.gz", None, "4.5.13", _SOURCE),
+ # A leading source/ dir is a build bucket, so a flat TLP release reads from the filename
+ ("myfaces", ("source",), "myfaces-1.0.9-src.tgz", None, "1.0.9", _SOURCE),
+ ("ant", ("source",), "apache-ant-1.10.0-src.tar.bz2", None, "1.10.0", _SOURCE),
+ # A named subproject still comes from the filename under the bucket (old jakarta naming)
+ ("struts", ("source",), "jakarta-struts-1.2.2-src.tar.gz", "jakarta-struts", "1.2.2", _SOURCE),
+ # A version dir inside source/ is ignored because the filename pins the version
+ ("couchdb", ("source", "1.0.3"), "apache-couchdb-1.0.3.tar.gz", None, "1.0.3", _SOURCE),
+ # A releases/ container is a bucket too
+ ("cloudstack", ("releases", "4.0.2"), "apache-cloudstack-4.0.2-src.tar.bz2", None, "4.0.2", _SOURCE),
+ # plugins/ is a bucket only under maven, where each plugin is its own release
+ ("maven", ("plugins",), "maven-acr-plugin-1.0-source-release.zip", "maven-acr-plugin", "1.0", _SOURCE),
+ # An underscore-separated version dir normalises to dots for the release key (the real
+ # path keeps its underscores, so downloads still resolve)
+ ("santuario", ("java-library", "1_4_7"), "xml-security-src-1_4_7.zip", "java-library", "1.4.7", _SOURCE),
+ ("ws", ("axis2", "0_92"), "Axis2Ideaplugin.jar", "axis2", "0.92", _BINARY),
+ # A build/status suffix that leaked into the filename name is stripped from the subproject
+ ("chukwa", (), "chukwa-src-0.6.0.tar.gz", None, "0.6.0", _SOURCE),
+ ("tapestry", (), "tapestry-src-5.0.10.tar.bz2", None, "5.0.10", _SOURCE),
+ ("chukwa", (), "chukwa-incubating-src-0.5.0.tar.gz", None, "0.5.0", _SOURCE),
+ # A v-prefixed version leaves a stray v on the name; strip it so the release merges with
+ # the non-v releases (apache- prefix is stripped later, at resolution)
+ (
+ "cloudstack",
+ ("releases", "kubernetes-provider-v1.0.0"),
+ "apache-cloudstack-kubernetes-provider-v1.0.0-src.tar.bz2",
+ "apache-cloudstack-kubernetes-provider",
+ "1.0.0",
+ _SOURCE,
+ ),
+]
+
+
+@pytest.mark.parametrize(("committee", "parts", "filename", "subproject", "version", "classification"), _CASES)
+def test_decompose_known_dist_layouts(
+ committee: str,
+ parts: tuple[str, ...],
+ filename: str | None,
+ subproject: str | None,
+ version: str | None,
+ classification: classify.FileType | None,
+) -> None:
+ if (filename is not None) and (classification is not None):
+ full_path = pathlib.PurePosixPath(committee, *parts, filename)
+ assert classify.classify_path(full_path) == classification
+ result = dist.decompose(committee, parts, filename)
+ assert result is not None
+ assert result.subproject == subproject
+ assert result.version == version
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org