You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tooling.apache.org by ar...@apache.org on 2026/07/28 17:30:25 UTC
(tooling-trusted-releases) branch arm updated: #1461 - rearrange catalog
This is an automated email from the ASF dual-hosted git repository.
alitheg pushed a commit to branch arm
in repository https://gitbox.apache.org/repos/asf/tooling-trusted-releases.git
The following commit(s) were added to refs/heads/arm by this push:
new ace7218a #1461 - rearrange catalog
ace7218a is described below
commit ace7218a3326ba8b07832eacba66bd20ac5aff5b
Author: Alastair McFarlane <ar...@apache.org>
AuthorDate: Tue Jul 28 18:30:05 2026 +0100
#1461 - rearrange catalog
---
atr/catalog_site.py | 144 +++++++++++++++++----
atr/templates/catalog_site/_project_back_link.html | 23 ++--
atr/templates/catalog_site/catalog_site_base.html | 1 +
atr/templates/catalog_site/committee.html | 47 -------
atr/templates/catalog_site/index.html | 2 +-
atr/templates/catalog_site/project.html | 60 ++++++---
6 files changed, 179 insertions(+), 98 deletions(-)
diff --git a/atr/catalog_site.py b/atr/catalog_site.py
index 576e464b..aa72d98a 100644
--- a/atr/catalog_site.py
+++ b/atr/catalog_site.py
@@ -30,10 +30,11 @@ import datetime
import json
import pathlib
import shutil
-from collections.abc import Sequence
+from collections.abc import Container, Iterable, Sequence
from typing import Any, Final
import aiofiles.os
+import aioshutil
import jinja2
import sqlmodel
@@ -113,6 +114,9 @@ async def generate_all(data: db.Session) -> None:
committees = await data.committee(_projects=True).all()
written = await _write_committee_pages(data, committees, site_dir)
await _write_index_pages(site_dir, written)
+ # Keyed off every committee rather than the ones written, so a committee whose page
+ # failed to render keeps what it had instead of losing the lot
+ await _prune_directories(site_dir, {committee.key for committee in committees} | {"assets"})
log.info(f"Rebuilt catalog site for {len(written)} of {len(committees)} committees")
@@ -152,15 +156,29 @@ async def regenerate_project(data: db.Session, project_key: str) -> None:
if project.committee_key is None:
log.warning(f"Catalog site: project {project_key} has no committee, skipping regeneration")
return
- # The project page links back past a committee that holds only the one project,
- # so it needs the committee's projects to know which way to point.
- committee = await data.committee(key=project.committee_key, _projects=True).get()
+ # The committee index is built from its projects, and the project page links back
+ # past a committee that holds only the one, so both need them loaded.
+ committees = await data.committee(_projects=True).all()
+ committee = next((c for c in committees if c.key == project.committee_key), None)
if committee is None:
log.warning(f"Catalog site: committee {project.committee_key} not found, skipping regeneration")
return
site_dir = paths.get_catalog_site_dir()
await _write_committee_index(data, committee, site_dir)
- await _write_project(data, committee, project, site_dir)
+ # A project sharing its committee's key is the committee page, so it's written already
+ if project.key != committee.key:
+ await _write_project(data, committee, project, site_dir)
+ # A first release, or a last one archived, moves a committee on or off the front page
+ # and between the Incubator's two columns, so the indexes are rebuilt every time too.
+ # The assets are not: they only change with a deploy, so the full rebuild owns them.
+ await _write_index_pages(site_dir, committees)
+
+
+def _last_updated() -> str:
+ return datetime.datetime.now(datetime.UTC).strftime("%d %b %Y %H:%M UTC")
+
+
+_ENVIRONMENT.globals["last_updated"] = _last_updated
async def _project_cle_document(data: db.Session, project: sql.Project, now: datetime.datetime) -> dict[str, Any]:
@@ -171,6 +189,49 @@ async def _project_cle_document(data: db.Session, project: sql.Project, now: dat
return cle.project_document(project, events, releases, now=now)
+def _project_path(committee: sql.Committee, project: sql.Project) -> str:
+ """Where a project's pages sit below its committee, as a relative path prefix.
+
+ A project sharing its committee's key is the committee page itself, so it gets no
+ directory of its own. The rest drop the committee key from the front of theirs,
+ unless a sibling is already keyed by what would be left.
+ """
+ if project.key == committee.key:
+ return ""
+ trimmed = project.key.removeprefix(f"{committee.key}-")
+ if any(sibling.key == trimmed for sibling in committee.projects):
+ return f"{project.key}/"
+ return f"{trimmed}/"
+
+
+_ENVIRONMENT.globals["project_path"] = _project_path
+
+
+def _project_segments(committee: sql.Committee, projects: Iterable[sql.Project]) -> set[str]:
+ return {_project_path(committee, project).removesuffix("/") for project in projects}
+
+
+async def _prune_directories(directory: safe.StatePath, keep: Container[str]) -> None:
+ """Remove the subdirectories of one that this build didn't write.
+
+ Files are left alone, since every page is rewritten in place. Only whole directories
+ go stale, when a committee, project or release stops being catalogued under that name.
+ """
+ path = directory.path
+ try:
+ entries = await aiofiles.os.listdir(path)
+ except FileNotFoundError:
+ return
+ pruned: list[str] = []
+ for name in sorted(entries):
+ if (name in keep) or (not await aiofiles.os.path.isdir(path / name)):
+ continue
+ await aioshutil.rmtree(path / name)
+ pruned.append(name)
+ if pruned:
+ log.info(f"Catalog site: pruned {len(pruned)} stale directories from {path}: {', '.join(pruned)}")
+
+
async def _write(path: safe.StatePath, content: str) -> None:
await util.atomic_write_file(path.path, content)
@@ -192,11 +253,11 @@ async def _write_attic_index(
# A retired PMC is filed either under the Attic itself or as a committee that kept
# its own key, but a reader wants one list, so the two are merged by name here.
# Nothing here still releases, so each entry opens straight at its archive.
- entries = [(project.display_name, f"{project.key}/archive.html") for project in projects]
+ entries = [(project.display_name, f"{_project_path(attic, project)}archive.html") for project in projects]
for committee in committees:
path = f"{committee.key}/index.html"
if len(committee.projects) == 1:
- path = f"{committee.key}/{committee.projects[0].key}/archive.html"
+ path = f"{committee.key}/{_project_path(committee, committee.projects[0])}archive.html"
entries.append((committee.display_name, f"../{path}"))
entries.sort(key=lambda entry: entry[0].lower())
html = _ENVIRONMENT.get_template("attic.html").render(committee=attic, entries=entries, root="../")
@@ -210,20 +271,33 @@ def _has_live_project(committee: sql.Committee) -> bool:
async def _write_committee_index(
data: db.Session, committee: sql.Committee, site_dir: safe.StatePath
) -> list[sql.Project]:
- projects = list(await data.project(committee_key=committee.key).all())
- current = sorted((p for p in projects if p.status in _LIVE_PROJECT_STATUSES), key=lambda p: p.display_name.lower())
- archived = sorted(
- (p for p in projects if p.status not in _LIVE_PROJECT_STATUSES), key=lambda p: p.display_name.lower()
- )
- html = _ENVIRONMENT.get_template("committee.html").render(
+ """Write the committee page, returning the projects that still need one of their own.
+
+ A committee running a project of its own name has no separate page for it. That
+ project's releases become the committee page, with the others listed beneath them.
+ """
+ projects = sorted(committee.projects, key=lambda project: project.display_name.lower())
+ tlp = next((project for project in projects if project.key == committee.key), None)
+ subprojects = [project for project in projects if project is not tlp]
+ current = [project for project in subprojects if project.status in _LIVE_PROJECT_STATUSES]
+ archived = [project for project in subprojects if project.status not in _LIVE_PROJECT_STATUSES]
+ if tlp is not None:
+ await _write_project(data, committee, tlp, site_dir, current, archived)
+ return subprojects
+ # Nothing of the committee's own to show, so the page is just the way down to
+ # the projects, rendered by the same template with no project against it.
+ html = _ENVIRONMENT.get_template("project.html").render(
committee=committee,
- current=current,
- archived=archived,
+ project=None,
+ subprojects=current,
+ archived_subprojects=archived,
retired=not current,
+ keys_url=paths.committee_keys_url(committee),
root="../",
)
await _write(site_dir / committee.key / "index.html", html)
- return projects
+ await _prune_directories(site_dir / committee.key, _project_segments(committee, subprojects))
+ return subprojects
async def _write_committee_pages(
@@ -245,6 +319,9 @@ async def _write_committee_pages(
await _write_project(data, committee, project, site_dir)
except Exception:
log.exception(f"Failed to render catalog site for project {project.key}")
+ if committee.key in _INDEXING_COMMITTEE_KEYS:
+ # These skip the committee index, which is where the tidying otherwise happens
+ await _prune_directories(site_dir / committee.key, _project_segments(committee, projects))
written.append(committee)
return written
@@ -299,7 +376,12 @@ async def _write_incubator_index(
async def _write_project(
- data: db.Session, committee: sql.Committee, project: sql.Project, site_dir: safe.StatePath
+ data: db.Session,
+ committee: sql.Committee,
+ project: sql.Project,
+ site_dir: safe.StatePath,
+ subprojects: Sequence[sql.Project] = (),
+ archived_subprojects: Sequence[sql.Project] = (),
) -> None:
now = datetime.datetime.now(datetime.UTC)
artifacts = await data.artifact(project_key=project.key, _release=True).all()
@@ -308,7 +390,12 @@ async def _write_project(
current = [v for v in assembled.versions if v.status == "released"]
archived = [v for v in assembled.versions if v.status == "archived"]
- project_dir = site_dir / committee.key / project.key
+ # A committee's own project is its index page, so it sits a level up from the
+ # rest and everything below it steps back one fewer time.
+ segment = _project_path(committee, project).removesuffix("/")
+ committee_dir = site_dir / committee.key
+ project_dir = (committee_dir / segment) if segment else committee_dir
+ root = "../../" if segment else "../"
retired = project.status not in _LIVE_PROJECT_STATUSES
# The signing keys are published per committee, beside the release files themselves
keys_url = paths.committee_keys_url(committee)
@@ -319,11 +406,13 @@ async def _write_project(
_ENVIRONMENT.get_template("project.html").render(
committee=committee,
project=project,
+ subprojects=subprojects,
+ archived_subprojects=archived_subprojects,
versions=current,
has_archive=bool(archived),
retired=retired,
keys_url=keys_url,
- root="../../",
+ root=root,
),
)
await _write(
@@ -334,15 +423,24 @@ async def _write_project(
versions=archived,
retired=retired,
keys_url=keys_url,
- root="../../",
+ root=root,
),
)
for version in assembled.versions:
- await _write_release(project_dir, committee, project, version)
+ await _write_release(project_dir, committee, project, version, f"{root}../")
+ keep = {str(version.version) for version in assembled.versions}
+ if not segment:
+ # Hoisted onto the committee page, so the subprojects sit beside the versions
+ keep |= _project_segments(committee, (*subprojects, *archived_subprojects))
+ await _prune_directories(project_dir, keep)
async def _write_release(
- project_dir: safe.StatePath, committee: sql.Committee, project: sql.Project, version: api.CatalogVersion
+ project_dir: safe.StatePath,
+ committee: sql.Committee,
+ project: sql.Project,
+ version: api.CatalogVersion,
+ root: str,
) -> None:
release_dir = project_dir / str(version.version)
await _write(
@@ -351,7 +449,7 @@ async def _write_release(
committee=committee,
project=project,
release_version=version,
- root="../../../",
+ root=root,
),
)
await _write(release_dir / "artifacts.json", version.model_dump_json(indent=2))
diff --git a/atr/templates/catalog_site/_project_back_link.html b/atr/templates/catalog_site/_project_back_link.html
index 439c69fa..2f8efdcf 100644
--- a/atr/templates/catalog_site/_project_back_link.html
+++ b/atr/templates/catalog_site/_project_back_link.html
@@ -1,14 +1,13 @@
-{# Shared by the project and archive pages, which sit at the same depth. The indexes
- link straight past a committee holding a single project, so step back the same way
- rather than to a committee page that was never passed through #}
-{% if committee.projects | length == 1 %}
- {% if committee.is_podling %}
- <p class="mb-1"><a href="{{ root }}{{ incubator_key }}/index.html" class="atr-back-link">← Incubator</a></p>
- {% elif retired %}
- <p class="mb-1"><a href="{{ root }}{{ attic_key }}/index.html" class="atr-back-link">← Attic</a></p>
- {% else %}
- <p class="mb-1"><a href="{{ root }}index.html" class="atr-back-link">← Release catalog</a></p>
- {% endif %}
-{% else %}
+{# The committee's own project is the committee page, and the indexes link straight past
+ a committee holding a single project, so both step back to an index rather than to a
+ committee page that was never passed through #}
+{% set own_page = project and (project.key != committee.key) %}
+{% if own_page and (committee.projects | length > 1) %}
<p class="mb-1"><a href="../index.html" class="atr-back-link">← {{ committee.display_name }}</a></p>
+{% elif committee.is_podling %}
+ <p class="mb-1"><a href="{{ root }}{{ incubator_key }}/index.html" class="atr-back-link">← Incubator</a></p>
+{% elif retired %}
+ <p class="mb-1"><a href="{{ root }}{{ attic_key }}/index.html" class="atr-back-link">← Attic</a></p>
+{% else %}
+ <p class="mb-1"><a href="{{ root }}index.html" class="atr-back-link">← Release catalog</a></p>
{% endif %}
diff --git a/atr/templates/catalog_site/catalog_site_base.html b/atr/templates/catalog_site/catalog_site_base.html
index d7969c48..2dc973a0 100644
--- a/atr/templates/catalog_site/catalog_site_base.html
+++ b/atr/templates/catalog_site/catalog_site_base.html
@@ -28,6 +28,7 @@
<main class="container py-4 flex-grow-1">
{% block content %}{% endblock content %}
+ <p class="text-muted small mt-4 mb-0">Last updated {{ last_updated() }}.</p>
</main>
{% include "includes/footer.html" %}
diff --git a/atr/templates/catalog_site/committee.html b/atr/templates/catalog_site/committee.html
deleted file mode 100644
index a8c95db1..00000000
--- a/atr/templates/catalog_site/committee.html
+++ /dev/null
@@ -1,47 +0,0 @@
-{% extends "catalog_site_base.html" %}
-
-{%- block title -%}{{ committee.display_name }} ~ Apache Release Catalog{%- endblock title -%}
-
-{%- block description -%}Releases for {{ committee.display_name }}.{%- endblock description -%}
-
-{% block content %}
- {# Podlings are catalogued on the Incubator page, not the front page #}
- {% if committee.is_podling %}
- <p class="mb-1"><a href="{{ root }}{{ incubator_key }}/index.html" class="atr-back-link">← Incubator</a></p>
- {% elif retired %}
- <p class="mb-1"><a href="{{ root }}{{ attic_key }}/index.html" class="atr-back-link">← Attic</a></p>
- {% else %}
- <p class="mb-1"><a href="{{ root }}index.html" class="atr-back-link">← Release catalog</a></p>
- {% endif %}
- <h1 class="mb-4">{{ committee.display_name }}</h1>
- {% if retired %}
- {% if not committee.is_podling %}
- <div class="alert alert-secondary" role="alert">
- This committee has retired to the Attic. Its projects are no longer active.
- </div>
- {% else %}
- <div class="alert alert-secondary" role="alert">
- This podling has retired from the Incubator. Its projects are no longer active.
- </div>
- {% endif %}
- {% else %}
- <h2 class="fs-4 border-bottom pb-1">Current projects</h2>
- {% if current %}
- <div class="list-group mb-4">
- {% for project in current %}
- <a class="list-group-item list-group-item-action" href="{{ project.key }}/index.html">{{ project.display_name }}</a>
- {% endfor %}
- </div>
- {% else %}
- <p class="text-muted">No current projects.</p>
- {% endif %}
- {% endif %}
- {% if archived %}
- <h2 class="fs-4 border-bottom pb-1">Archived projects</h2>
- <div class="list-group">
- {% for project in archived %}
- <a class="list-group-item list-group-item-action" href="{{ project.key }}/index.html">{{ project.display_name }}</a>
- {% endfor %}
- </div>
- {% endif %}
-{% endblock content %}
diff --git a/atr/templates/catalog_site/index.html b/atr/templates/catalog_site/index.html
index 241e230d..f29b72f1 100644
--- a/atr/templates/catalog_site/index.html
+++ b/atr/templates/catalog_site/index.html
@@ -18,7 +18,7 @@
{# A committee with a single project has nothing to choose from, so
the card opens the project itself #}
{% if committee.projects | length == 1 %}
- {% set target = committee.key ~ "/" ~ committee.projects[0].key ~ "/index.html" %}
+ {% set target = committee.key ~ "/" ~ project_path(committee, committee.projects[0]) ~ "index.html" %}
{% else %}
{% set target = committee.key ~ "/index.html" %}
{% endif %}
diff --git a/atr/templates/catalog_site/project.html b/atr/templates/catalog_site/project.html
index f6f3e7af..14b2f29d 100644
--- a/atr/templates/catalog_site/project.html
+++ b/atr/templates/catalog_site/project.html
@@ -1,34 +1,64 @@
{% extends "catalog_site_base.html" %}
-{%- block title -%}{{ project.display_name }} ~ Apache Release Catalog{%- endblock title -%}
+{% set heading = project.display_name if project else committee.display_name %}
-{%- block description -%}Release catalogue for {{ project.display_name }}.{%- endblock description -%}
+{%- block title -%}{{ heading }} ~ Apache Release Catalog{%- endblock title -%}
+
+{%- block description -%}Release catalogue for {{ heading }}.{%- endblock description -%}
{% block content %}
{% include "_project_back_link.html" %}
<div class="d-flex justify-content-between align-items-center flex-wrap gap-2 mb-3">
- <h1 class="mb-0">{{ project.display_name }}</h1>
+ <h1 class="mb-0">{{ heading }}</h1>
<span>
<a class="btn btn-sm btn-outline-secondary" href="{{ keys_url }}">Public KEYS</a>
- <a class="btn btn-sm btn-outline-secondary ms-2" href="cle.json">Project CLE</a>
- {# A retired project has nothing current, so the archive is always the way on #}
- {% if has_archive or retired %}
- <a class="btn btn-sm btn-outline-secondary ms-2" href="archive.html">Archived releases</a>
+ {% if project %}
+ <a class="btn btn-sm btn-outline-secondary ms-2" href="cle.json">Project CLE</a>
+ {# A retired project has nothing current, so the archive is always the way on #}
+ {% if has_archive or retired %}
+ <a class="btn btn-sm btn-outline-secondary ms-2" href="archive.html">Archived releases</a>
+ {% endif %}
{% endif %}
</span>
</div>
{% if retired %}
<div class="alert alert-secondary" role="alert">
- This project has been archived. See its <a href="archive.html" class="alert-link">archived releases</a> for the history.
+ {% if project %}
+ This project has been archived. See its <a href="archive.html" class="alert-link">archived releases</a> for the history.
+ {% elif committee.is_podling %}
+ This podling has retired from the Incubator. Its projects are no longer active.
+ {% else %}
+ This committee has retired to the Attic. Its projects are no longer active.
+ {% endif %}
+ </div>
+ {% endif %}
+ {% if project %}
+ {# A retired project can still hold releases that were never archived themselves.
+ They belong to nothing else, so list them here rather than drop them #}
+ {% if versions %}
+ {% for version in versions %}{% include "_version.html" %}{% endfor %}
+ {% elif not retired %}
+ <div class="alert alert-info" role="alert">
+ No catalogued releases yet for {{ heading }}.
+ </div>
+ {% endif %}
+ {% endif %}
+ {% if subprojects %}
+ <h2 class="fs-4 border-bottom pb-1 mt-4">Subprojects</h2>
+ <div class="list-group mb-4">
+ {% for subproject in subprojects %}
+ <a class="list-group-item list-group-item-action"
+ href="{{ project_path(committee, subproject) }}index.html">{{ subproject.display_name }}</a>
+ {% endfor %}
</div>
{% endif %}
- {# A retired project can still hold releases that were never archived themselves.
- They belong to nothing else, so list them here rather than drop them #}
- {% if versions %}
- {% for version in versions %}{% include "_version.html" %}{% endfor %}
- {% elif not retired %}
- <div class="alert alert-info" role="alert">
- No catalogued releases yet for {{ project.display_name }}.
+ {% if archived_subprojects %}
+ <h2 class="fs-4 border-bottom pb-1 mt-4">Archived subprojects</h2>
+ <div class="list-group">
+ {% for subproject in archived_subprojects %}
+ <a class="list-group-item list-group-item-action"
+ href="{{ project_path(committee, subproject) }}index.html">{{ subproject.display_name }}</a>
+ {% endfor %}
</div>
{% endif %}
{% endblock content %}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org