You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tooling.apache.org by ar...@apache.org on 2026/07/28 15:34:09 UTC

(tooling-trusted-releases) branch arm updated (a9c28522 -> f50486b8)

This is an automated email from the ASF dual-hosted git repository.

alitheg pushed a change to branch arm
in repository https://gitbox.apache.org/repos/asf/tooling-trusted-releases.git


 discard a9c28522 Fix on dist watcher for whole-directory adds
     add 49df4d73 Add trademarks to the footer (#1459)
     add 4255139d Reuse app footer
     new f50486b8 Fix on dist watcher for whole-directory adds

This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version.  This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:

 * -- * -- B -- O -- O -- O   (a9c28522)
            \
             N -- N -- N   refs/heads/arm (f50486b8)

You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.

Any revisions marked "omit" are not gone; other references still
refer to them.  Any revisions marked "discard" are gone forever.

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 atr/catalog_site.py                               |  9 ++++++++-
 atr/templates/catalog_site/catalog_site_base.html | 12 +-----------
 atr/templates/includes/footer.html                |  5 +++++
 3 files changed, 14 insertions(+), 12 deletions(-)


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org


(tooling-trusted-releases) 01/01: Fix on dist watcher for whole-directory adds

Posted by ar...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

alitheg pushed a commit to branch arm
in repository https://gitbox.apache.org/repos/asf/tooling-trusted-releases.git

commit f50486b84d194db9d12b84d57c8fd10f78f704df
Author: Alastair McFarlane <ar...@apache.org>
AuthorDate: Tue Jul 28 16:33:47 2026 +0100

    Fix on dist watcher for whole-directory adds
---
 atr/svn/__init__.py |  7 +++++++
 atr/svn/catalog.py  | 60 +++++++++++++++++++++++++++++++++++++++++++++--------
 2 files changed, 58 insertions(+), 9 deletions(-)

diff --git a/atr/svn/__init__.py b/atr/svn/__init__.py
index de11a144..ca7af5eb 100644
--- a/atr/svn/__init__.py
+++ b/atr/svn/__init__.py
@@ -32,6 +32,7 @@ ASF_TOOL: Final[str] = "atr"
 EXPORT_TIMEOUT_SECONDS: Final[float] = 240.0
 INFO_TIMEOUT_SECONDS: Final[float] = 30.0
 KEYS_TIMEOUT_SECONDS: Final[float] = 60.0
+LIST_TIMEOUT_SECONDS: Final[float] = 60.0
 PUBLISH_TIMEOUT_SECONDS: Final[float] = 240.0
 _COMMITTED_REVISION_RE: Final = re.compile(r"^Committed revision (\d+)\.\s*$", re.MULTILINE)
 _CONNECTION_ERROR_CODES: Final[frozenset[str]] = frozenset(
@@ -196,6 +197,12 @@ async def get_log(path: pathlib.Path) -> SvnLog:
     return SvnLog.from_xml_tree(root)
 
 
+async def list_files(url: str) -> list[str]:
+    """List every file below a URL, as paths relative to it. Directories are left out."""
+    output = await _run_svn_command("list", url, "--recursive", timeout_seconds=LIST_TIMEOUT_SECONDS)
+    return [line for line in output.splitlines() if line and (not line.endswith("/"))]
+
+
 def parse_committed_revision(output: str) -> int | None:
     if (match := _COMMITTED_REVISION_RE.search(output)) is None:
         return None
diff --git a/atr/svn/catalog.py b/atr/svn/catalog.py
index b48ffdf2..6329775f 100644
--- a/atr/svn/catalog.py
+++ b/atr/svn/catalog.py
@@ -22,11 +22,12 @@ Catalogue published dist commits into the database via the storage interface.
 import dataclasses
 import datetime
 import pathlib
-from typing import Final
+from typing import Final, NamedTuple
 
 import atr.analysis as analysis
 import atr.classify as classify
 import atr.config as config
+import atr.constants as constants
 import atr.db as db
 import atr.log as log
 import atr.models.safe as safe
@@ -48,6 +49,9 @@ _CHECKSUM_SUFFIXES: Final[tuple[str, ...]] = (".sha512", ".sha256", ".sha1", ".s
 type _ResolvedRelease = tuple[safe.ProjectKey, safe.VersionKey, list[release.ArtifactInput]]
 type _ResolvedArchive = tuple[safe.ProjectKey, safe.VersionKey]
 
+# How a release is identified while a commit is being read: committee, subproject, version
+type _ReleaseKey = tuple[str, str | None, str]
+
 
 @dataclasses.dataclass
 class _ReleaseFiles:
@@ -60,6 +64,14 @@ class _ReleaseFiles:
     has_source: bool = False
 
 
+class _StructuralChanges(NamedTuple):
+    # Releases whose files the commit lists, version directories it deleted, and version
+    # directories it added whole, mapped to their path under the release root
+    added: dict[_ReleaseKey, _ReleaseFiles]
+    removed: set[_ReleaseKey]
+    copied: dict[_ReleaseKey, str]
+
+
 async def catalogue_commit(commit: dict) -> None:
     if str(commit.get("committer", "")) == svn.ASF_TOOL:
         # Our own publishes are already in the database
@@ -68,7 +80,9 @@ async def catalogue_commit(commit: dict) -> None:
     if not isinstance(changed, dict):
         log.warning(f"dist commit payload has unexpected changed shape: {type(changed).__name__}")
         return
-    added, removed = _structural_changes(changed)
+    changes = _structural_changes(changed)
+    added, removed = changes.added, changes.removed
+    await _expand_copied(changes.copied, added)
     added = _collapse_airflow_providers(added)
     if not (added or removed):
         return
@@ -161,6 +175,31 @@ def _companion(siblings: set[str], artifact: str, suffixes: tuple[str, ...]) ->
     return None
 
 
+async def _expand_copied(copied: dict[_ReleaseKey, str], added: dict[_ReleaseKey, _ReleaseFiles]) -> None:
+    # Publishing by copying a whole version directory across from dev is the common
+    # release flow, and svn reports it as a single added directory. The files have to be
+    # listed from the repository, since the commit itself never names them
+    for key, relpath in copied.items():
+        if key in added:
+            continue
+        committee, subproject, version = key
+        try:
+            names = await svn.list_files(f"{constants.SVN_DIST_ROOT_URL}/{_RELEASE_PREFIX}{relpath}")
+        except Exception:
+            log.exception(f"dist watcher could not list {relpath}, so it stays uncatalogued")
+            continue
+        bundle = _ReleaseFiles(committee, subproject, version)
+        for name in names:
+            rel = f"{relpath}/{name}"
+            dirpath, _, filename = rel.rpartition("/")
+            file_type = classify.classify_path(pathlib.PurePosixPath(rel))
+            bundle.files.append((dirpath, filename, file_type))
+            if file_type is classify.FileType.SOURCE:
+                bundle.has_source = True
+        if bundle.has_source:
+            added[key] = bundle
+
+
 def _decompose_change(path: str) -> tuple[str, dist.Decomposed, bool, str | None] | None:
     # release/<committee>/... only; returns committee, decomposition, is_dir and
     # the filename (None for a directory)
@@ -288,14 +327,14 @@ def _sbom_companion(siblings: set[str], artifact: str) -> str | None:
     return None
 
 
-def _structural_changes(
-    changed: dict,
-) -> tuple[dict[tuple[str, str | None, str], _ReleaseFiles], set[tuple[str, str | None, str]]]:
+def _structural_changes(changed: dict) -> _StructuralChanges:
     # Group added files by release so we can record their artifacts, and collect
     # version-dir deletions as archivals. A release is only identified by an added
-    # source artifact; binary/doc/metadata files are included but don't make a release
-    added: dict[tuple[str, str | None, str], _ReleaseFiles] = {}
-    removed: set[tuple[str, str | None, str]] = set()
+    # source artifact; binary/doc/metadata files are included but don't make a release.
+    # A version directory added whole is kept aside, since its files are not in the commit
+    added: dict[_ReleaseKey, _ReleaseFiles] = {}
+    removed: set[_ReleaseKey] = set()
+    copied: dict[_ReleaseKey, str] = {}
     for raw_path, info in changed.items():
         path = str(raw_path)
         flags = str(info.get("flags", "")) if isinstance(info, dict) else ""
@@ -310,6 +349,8 @@ def _structural_changes(
         key = (committee, subproject, decomposed.version)
         if flags.startswith("D") and is_dir:
             removed.add(key)
+        elif flags.startswith("A") and is_dir:
+            copied[key] = path.removeprefix(_RELEASE_PREFIX).rstrip("/")
         elif flags.startswith("A") and (filename is not None):
             bundle = added.get(key)
             if bundle is None:
@@ -321,7 +362,8 @@ def _structural_changes(
             bundle.files.append((dirpath, filename, file_type))
             if file_type is classify.FileType.SOURCE:
                 bundle.has_source = True
-    return {key: bundle for key, bundle in added.items() if bundle.has_source}, removed
+    with_source = {key: bundle for key, bundle in added.items() if bundle.has_source}
+    return _StructuralChanges(added=with_source, removed=removed, copied=copied)
 
 
 def _airflow_bundle_area(bundle: _ReleaseFiles) -> str | None:


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org