You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tooling.apache.org by sb...@apache.org on 2026/09/02 14:08:10 UTC
(tooling-trusted-releases) branch sbp updated (b9f9c0e0 -> 57328e2f)
This is an automated email from the ASF dual-hosted git repository.
sbp pushed a change to branch sbp
in repository https://gitbox.apache.org/repos/asf/tooling-trusted-releases.git
from b9f9c0e0 Paginate a page and API endpoint listing releases
add 1839e8a1 #1544 - fix classifier-style SBOM matching
new 57328e2f Paginate artifacts on dynamic catalogue pages
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
atr/analysis.py | 24 +++++++++++
atr/db/__init__.py | 4 ++
atr/db/interaction.py | 20 +++++++++
atr/get/catalog.py | 47 +++++++++++++++-----
atr/shared/catalog.py | 86 +++++++++++++++++++++++++++++++------
atr/tasks/checks/paths.py | 10 ++---
atr/templates/_catalog_version.html | 3 ++
atr/templates/catalog.html | 40 +++++++++++++++--
tests/unit/test_catalog.py | 62 ++++++++++++++++++++++++++
tests/unit/test_sbom_candidates.py | 39 +++++++++++++++++
10 files changed, 303 insertions(+), 32 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org
(tooling-trusted-releases) 01/01: Paginate artifacts on dynamic catalogue pages
Posted by sb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sbp pushed a commit to branch sbp
in repository https://gitbox.apache.org/repos/asf/tooling-trusted-releases.git
commit 57328e2f450ad09b924b6d32739c1a15afccf906
Author: Sean B. Palmer <se...@miscoranda.com>
AuthorDate: Wed Sep 2 15:07:09 2026 +0100
Paginate artifacts on dynamic catalogue pages
---
atr/db/__init__.py | 4 ++
atr/db/interaction.py | 20 +++++++++
atr/get/catalog.py | 47 +++++++++++++++-----
atr/shared/catalog.py | 86 +++++++++++++++++++++++++++++++------
atr/templates/_catalog_version.html | 3 ++
atr/templates/catalog.html | 40 +++++++++++++++--
tests/unit/test_catalog.py | 62 ++++++++++++++++++++++++++
7 files changed, 236 insertions(+), 26 deletions(-)
diff --git a/atr/db/__init__.py b/atr/db/__init__.py
index f1ece8cb..95411903 100644
--- a/atr/db/__init__.py
+++ b/atr/db/__init__.py
@@ -210,6 +210,7 @@ class Session(sqlalchemy.ext.asyncio.AsyncSession):
self,
project_key: Opt[str] = NOT_SET,
version: Opt[str] = NOT_SET,
+ version_in: Opt[Sequence[str]] = NOT_SET,
artifact_path: Opt[str] = NOT_SET,
release_key: Opt[str | None] = NOT_SET,
key_fingerprint: Opt[str | None] = NOT_SET,
@@ -222,11 +223,14 @@ class Session(sqlalchemy.ext.asyncio.AsyncSession):
_release: bool = False,
) -> Query[sql.Artifact]:
query = sqlmodel.select(sql.Artifact)
+ via = sql.validate_instrumented_attribute
if is_defined(project_key):
query = query.where(sql.Artifact.project_key == project_key)
if is_defined(version):
query = query.where(sql.Artifact.version == version)
+ if is_defined(version_in):
+ query = query.where(via(sql.Artifact.version).in_(version_in))
if is_defined(artifact_path):
query = query.where(sql.Artifact.artifact_path == artifact_path)
if is_defined(release_key):
diff --git a/atr/db/interaction.py b/atr/db/interaction.py
index 5333f7aa..10086be0 100644
--- a/atr/db/interaction.py
+++ b/atr/db/interaction.py
@@ -154,6 +154,26 @@ async def all_releases(project: sql.Project) -> list[sql.Release]:
return results
+async def catalog_version_rows(
+ data: db.Session, project_key: str
+) -> list[tuple[str, datetime.datetime | None, int | None, int]]:
+ via = sql.validate_instrumented_attribute
+ released = sqlalchemy.func.coalesce(via(sql.Release.released), via(sql.Release.created))
+ rows = await data.execute(
+ sqlalchemy.select(
+ via(sql.Artifact.version),
+ sqlalchemy.func.max(released),
+ sqlalchemy.func.max(via(sql.Artifact.svn_revision)),
+ sqlalchemy.func.count(),
+ )
+ .select_from(sql.Artifact)
+ .outerjoin(sql.Release, via(sql.Release.key) == via(sql.Artifact.release_key))
+ .where(via(sql.Artifact.project_key) == project_key)
+ .group_by(via(sql.Artifact.version))
+ )
+ return [(row[0], row[1], row[2], row[3]) for row in rows.all()]
+
+
async def certificate_artifact_counts(data: db.Session, fingerprints: list[str]) -> dict[str, int]:
# How many catalogued artifacts each certificate has signed, keyed by certificate fingerprint.
# An artifact is attributed to the key which signed it, so counting per certificate has to go
diff --git a/atr/get/catalog.py b/atr/get/catalog.py
index 2271e2df..f08ef7db 100644
--- a/atr/get/catalog.py
+++ b/atr/get/catalog.py
@@ -15,7 +15,9 @@
# specific language governing permissions and limitations
# under the License.
+import dataclasses
import datetime
+from collections.abc import Sequence
from typing import Literal
import asfquart.base as base
@@ -23,41 +25,66 @@ import asfquart.base as base
import atr.api as api
import atr.blueprints.get as get
import atr.db as db
+import atr.db.interaction as interaction
import atr.models.safe as safe
+import atr.models.sql as sql
+import atr.models.validation as validation
import atr.shared.catalog as catalog
import atr.template as template
import atr.util as util
import atr.web as web
+@dataclasses.dataclass
+class CatalogQuery(web.PageQuery):
+ limit: int = 200
+
+
@get.typed
-async def project(_session: web.Public, _catalog: Literal["catalog"], project_key: safe.ProjectKey) -> str:
+async def project(
+ _session: web.Public, _catalog: Literal["catalog"], project_key: safe.ProjectKey, query_args: CatalogQuery
+) -> str:
"""
URL: /catalog/<project_key>
The release catalogue for a project: versions and their artifacts.
"""
+ try:
+ validation.pagination_args_validate(query_args)
+ except ValueError as e:
+ raise base.ASFQuartException(str(e), errorcode=400)
async with db.session() as data:
project_obj = await data.project(key=str(project_key)).demand(
base.ASFQuartException(f"Project {project_key} not found", errorcode=404)
)
- artifacts = await data.artifact(project_key=project_obj.key, _release=True).all()
project_cycles = await data.project_cycle(project_key=project_obj.key).all()
+ rows = await interaction.catalog_version_rows(data, project_obj.key)
+ window = catalog.artifact_window(project_obj.version_method, rows, query_args.offset, query_args.limit)
+ artifacts = await _page_artifacts(data, project_obj.key, window)
now = datetime.datetime.now(datetime.UTC)
- assembled = catalog.assemble(
- project_obj.version_method,
- artifacts,
- project_cycles,
- now,
- )
+ assembled = catalog.assemble(project_obj.version_method, artifacts, project_cycles, now)
+ versions = catalog.clip_versions(assembled.versions, window, query_args.limit)
+ shown = sum(len(entry.artifacts) for entry in versions)
+ page = web.page_nav(query_args.offset, query_args.limit, window.count, shown)
+ groups = catalog.cycle_groups(versions, project_cycles, now, order=window.versions) if assembled.grouped else []
return await template.render(
"catalog.html",
project=project_obj,
committee=project_obj.committee,
- versions=assembled.versions,
- groups=catalog.cycle_groups(assembled.versions, project_cycles, now) if assembled.grouped else [],
+ versions=versions,
+ groups=groups,
grouped=assembled.grouped,
api=api,
format_datetime=util.format_datetime,
+ count=window.count,
+ limit=query_args.limit,
+ page=page,
+ continued_version=window.versions[0] if window.skip else None,
)
+
+
+async def _page_artifacts(data: db.Session, project_key: str, window: catalog.ArtifactWindow) -> Sequence[sql.Artifact]:
+ if not window.versions:
+ return []
+ return await data.artifact(project_key=project_key, version_in=window.versions, _release=True).all()
diff --git a/atr/shared/catalog.py b/atr/shared/catalog.py
index bc476dab..f5298e8a 100644
--- a/atr/shared/catalog.py
+++ b/atr/shared/catalog.py
@@ -32,6 +32,13 @@ import atr.util as util
_GROUPED_METHODS: Final[frozenset[sql.VersionMethod]] = frozenset({sql.VersionMethod.SEMVER, sql.VersionMethod.CALVER})
+@dataclasses.dataclass
+class ArtifactWindow:
+ versions: list[str]
+ skip: int
+ count: int
+
+
@dataclasses.dataclass
class CatalogProject:
versions: list[models.api.CatalogVersion]
@@ -52,6 +59,29 @@ class CycleGroup:
versions: Sequence[models.api.CatalogVersion]
+def artifact_window(
+ version_method: sql.VersionMethod,
+ rows: Sequence[tuple[str, datetime.datetime | None, int | None, int]],
+ offset: int,
+ limit: int,
+) -> ArtifactWindow:
+ counts = {row[0]: row[3] for row in rows}
+ versions: list[str] = []
+ skip = 0
+ end = 0
+ for version_key in version_order(version_method, [row[:3] for row in rows]):
+ start = end
+ end += counts[version_key]
+ if end <= offset:
+ continue
+ if start >= (offset + limit):
+ break
+ if not versions:
+ skip = offset - start
+ versions.append(version_key)
+ return ArtifactWindow(versions=versions, skip=skip, count=sum(counts.values()))
+
+
def assemble(
version_method: sql.VersionMethod,
artifacts: Sequence[sql.Artifact],
@@ -81,10 +111,27 @@ def cle_release_url(atr_host: str, project_key: str, version: str) -> str:
return f"https://{atr_host}/api/cle/release/{project_key}/{version}"
+def clip_versions(
+ versions: Sequence[models.api.CatalogVersion], window: ArtifactWindow, limit: int
+) -> list[models.api.CatalogVersion]:
+ ranks = {key: index for index, key in enumerate(window.versions)}
+ clipped = sorted(versions, key=lambda entry: ranks[str(entry.version)])
+ skip = window.skip
+ remaining = limit
+ for entry in clipped:
+ artifacts = sorted(entry.artifacts, key=lambda artifact: artifact.artifact_path)
+ entry.artifacts = artifacts[skip : (skip + remaining)]
+ remaining -= len(entry.artifacts)
+ skip = 0
+ return clipped
+
+
def cycle_groups(
versions: Sequence[models.api.CatalogVersion],
project_cycles: Sequence[sql.ProjectCycle],
now: datetime.datetime,
+ *,
+ order: Sequence[str] | None = None,
) -> list[CycleGroup]:
"""Group versions into their cycles for display, most recently active first.
@@ -93,6 +140,9 @@ def cycle_groups(
releases on one page, archives on another - labels each side by the same rule.
"""
grouped = _cycles(versions, {cycle.cycle: cycle for cycle in project_cycles}, now)
+ if order is not None:
+ ranks = {key: index for index, key in enumerate(order)}
+ grouped.sort(key=lambda entry: ranks[str(entry.versions[0].version)])
headed = cycles.headings_needed(entry.cycle for entry in grouped)
return [
CycleGroup(
@@ -104,6 +154,16 @@ def cycle_groups(
]
+def version_order(
+ version_method: sql.VersionMethod, items: Iterable[tuple[str, datetime.datetime | None, int | None]]
+) -> list[str]:
+ entries = list(items)
+ if _grouped_layout(version_method):
+ ranks = _version_ranks(entry[0] for entry in entries)
+ return [entry[0] for entry in sorted(entries, key=lambda entry: ranks[entry[0]], reverse=True)]
+ return [entry[0] for entry in sorted(entries, key=_version_date_key, reverse=True)]
+
+
def _artifact(row: sql.Artifact, downloadable: bool, archived: bool) -> models.api.CatalogArtifact:
artifact_url: str | None = None
signature_url: str | None = None
@@ -203,24 +263,24 @@ def _loose_version_key(text: str) -> tuple[tuple[int, int | str], ...]:
return tuple(parts)
+def _version_date_key(entry: tuple[str, datetime.datetime | None, int | None]) -> tuple[datetime.datetime, int, str]:
+ # Newest first: released date leads, svn revision breaks ties for older rows.
+ released = entry[1] or datetime.datetime.min.replace(tzinfo=datetime.UTC)
+ return (released, entry[2] or 0, entry[0])
+
+
def _version_ranks(labels: Iterable[str]) -> dict[str, int]:
# Rank distinct version strings (or cycle labels, which are version prefixes) low to high, so a
# caller sorts newest first by reversing. PEP 440 first; one string it rejects drops the whole set
# to the loose split, so an odd tag can't throw the order or raise
distinct = list(set(labels))
try:
- ordered = sorted(distinct, key=version.Version)
+ ordered = sorted(distinct, key=lambda label: (version.Version(label), label))
except version.InvalidVersion:
- ordered = sorted(distinct, key=_loose_version_key)
+ ordered = sorted(distinct, key=lambda label: (_loose_version_key(label), label))
return {label: index for index, label in enumerate(ordered)}
-def _version_sort_key(entry: models.api.CatalogVersion) -> tuple[datetime.datetime, int]:
- # Newest first: released date leads, svn revision breaks ties for older rows.
- released = entry.released or datetime.datetime.min.replace(tzinfo=datetime.UTC)
- return (released, entry.svn_revision or 0)
-
-
def _versions(
artifacts: Sequence[sql.Artifact],
cycles_by_key: dict[str, sql.ProjectCycle],
@@ -262,9 +322,9 @@ def _versions(
# A project with a version scheme (semver/calver) orders by the version itself, so a later patch
# of an older line can't jump a newer line the way a date sort lets it. A simple project has no
# scheme, so it keeps the by-date order its lone default cycle wants
- if _grouped_layout(version_method):
- ranks = _version_ranks(str(entry.version) for entry in versions)
- versions.sort(key=lambda entry: ranks[str(entry.version)], reverse=True)
- else:
- versions.sort(key=_version_sort_key, reverse=True)
+ order = version_order(
+ version_method, [(str(entry.version), entry.released, entry.svn_revision) for entry in versions]
+ )
+ ranks = {key: index for index, key in enumerate(order)}
+ versions.sort(key=lambda entry: ranks[str(entry.version)])
return versions
diff --git a/atr/templates/_catalog_version.html b/atr/templates/_catalog_version.html
index 3254d0a1..9e13a589 100644
--- a/atr/templates/_catalog_version.html
+++ b/atr/templates/_catalog_version.html
@@ -9,6 +9,9 @@
<div class="card-body">
<div class="d-flex justify-content-between align-items-center mb-1">
<strong class="fs-5">{{ version.version }}</strong>
+ {% if version.version|string == continued_version %}
+ <span class="badge text-bg-light">Continued</span>
+ {% endif %}
<span>
<span class="badge {{ status_classes[version.status] }}">{{ version.status | capitalize }}</span>
{% if version.vote_thread_url %}
diff --git a/atr/templates/catalog.html b/atr/templates/catalog.html
index 386b9d47..62b9af92 100644
--- a/atr/templates/catalog.html
+++ b/atr/templates/catalog.html
@@ -21,10 +21,20 @@
</div>
{% endif %}
+ {% if versions %}
+ <p>Showing artifacts {{ page.start }} to {{ page.end }} of {{ count }}.</p>
+ {% endif %}
+
{% if not versions %}
- <div class="alert alert-info mt-3" role="alert">
- <i class="bi bi-info-circle me-2"></i> No catalogued releases yet for {{ project.display_name }}.
- </div>
+ {% if count %}
+ <div class="alert alert-info mt-3" role="alert">
+ <i class="bi bi-info-circle me-2"></i> No artifacts to show on this page ({{ count }} total).
+ </div>
+ {% else %}
+ <div class="alert alert-info mt-3" role="alert">
+ <i class="bi bi-info-circle me-2"></i> No catalogued releases yet for {{ project.display_name }}.
+ </div>
+ {% endif %}
{% elif grouped %}
{% for group in groups %}
<div class="mb-4">
@@ -41,6 +51,7 @@
{% endfor %}
{% else %}
<div class="mb-3" id="catalog-filter">
+ <span class="text-muted small me-2">This page:</span>
<button type="button" class="btn btn-sm btn-primary active" data-status-filter="all">All</button>
<button type="button" class="btn btn-sm btn-outline-secondary" data-status-filter="released">Released</button>
<button type="button" class="btn btn-sm btn-outline-secondary" data-status-filter="archived">Archived</button>
@@ -49,6 +60,29 @@
{% include "_catalog_version.html" %}
{% endfor %}
{% endif %}
+
+ {% if (page.previous_offset is not none) or (page.next_offset is not none) %}
+ <nav aria-label="Catalog pages">
+ <ul class="pagination">
+ {% if page.previous_offset is not none %}
+ <li class="page-item">
+ <a class="page-link"
+ href="{{ as_url(get.catalog.project, project_key=project.key, limit=limit, offset=page.previous_offset) }}">Previous</a>
+ </li>
+ {% else %}
+ <li class="page-item disabled"><span class="page-link">Previous</span></li>
+ {% endif %}
+ {% if page.next_offset is not none %}
+ <li class="page-item">
+ <a class="page-link"
+ href="{{ as_url(get.catalog.project, project_key=project.key, limit=limit, offset=page.next_offset) }}">Next</a>
+ </li>
+ {% else %}
+ <li class="page-item disabled"><span class="page-link">Next</span></li>
+ {% endif %}
+ </ul>
+ </nav>
+ {% endif %}
{% endblock content %}
{% block javascripts %}
diff --git a/tests/unit/test_catalog.py b/tests/unit/test_catalog.py
index 98b02a6b..8c72e3fd 100644
--- a/tests/unit/test_catalog.py
+++ b/tests/unit/test_catalog.py
@@ -51,6 +51,30 @@ def test_archived_status_holds_without_an_archive_date() -> None:
assert versions["3.11.0"].artifacts[0].downloadable is True
+def test_artifact_window_clips_at_an_exact_version_boundary() -> None:
+ rows = [("1.0", _NOW, None, 2), ("2.0", _NOW + datetime.timedelta(days=1), None, 3)]
+
+ window = catalog.artifact_window(sql.VersionMethod.SIMPLE, rows, 0, 3)
+
+ assert window == catalog.ArtifactWindow(versions=["2.0"], skip=0, count=5)
+
+
+def test_artifact_window_is_empty_beyond_the_end() -> None:
+ rows = [("1.0", _NOW, None, 2)]
+
+ window = catalog.artifact_window(sql.VersionMethod.SIMPLE, rows, 2, 200)
+
+ assert window == catalog.ArtifactWindow(versions=[], skip=0, count=2)
+
+
+def test_artifact_window_splits_a_dense_version_with_a_skip() -> None:
+ rows = [("1.0", _NOW, None, 2), ("2.0", _NOW + datetime.timedelta(days=1), None, 5)]
+
+ window = catalog.artifact_window(sql.VersionMethod.SIMPLE, rows, 4, 2)
+
+ assert window == catalog.ArtifactWindow(versions=["2.0", "1.0"], skip=4, count=7)
+
+
def test_cle_url_is_omitted_for_versions_without_a_backing_release() -> None:
project = _project()
artifacts = [_artifact(project, "3.0.0", "a-3.0.0.tar.gz", svn_revision=100)]
@@ -85,6 +109,26 @@ def test_cle_url_links_versions_backed_by_a_release_when_a_host_is_supplied() ->
assert versions["5.0.2"].cle_url == "https://atr.example.org/api/cle/release/cassandra/5.0.2"
+def test_clip_versions_follows_window_order_and_keeps_header_metadata() -> None:
+ project = _project()
+ release = _release(project, "2.0", cycle_key="cassandra-default", released=_NOW)
+ artifacts = [
+ _artifact(project, "1.0", "a-1.0.tar.gz", svn_revision=10),
+ _artifact(project, "2.0", "b-2.0.tar.gz", release=release, managed=True),
+ _artifact(project, "2.0", "a-2.0.tar.gz", release=release, svn_revision=20),
+ ]
+ assembled = catalog.assemble(sql.VersionMethod.SIMPLE, artifacts, [], _NOW)
+ window = catalog.ArtifactWindow(versions=["2.0", "1.0"], skip=1, count=3)
+
+ clipped = catalog.clip_versions(assembled.versions, window, 2)
+
+ assert [str(entry.version) for entry in clipped] == ["2.0", "1.0"]
+ assert [artifact.artifact_path for artifact in clipped[0].artifacts] == ["b-2.0.tar.gz"]
+ assert [artifact.artifact_path for artifact in clipped[1].artifacts] == ["a-1.0.tar.gz"]
+ assert clipped[0].managed is True
+ assert clipped[0].svn_revision == 20
+
+
def test_cycles_are_ordered_by_version_not_activity() -> None:
# Cycles order by the version their label carries, newest first, not by activity date: a late
# patch on the older 4.1 line post-dates 5.0 here, but 5.0 is the newer line, so it still leads
@@ -201,6 +245,24 @@ def test_version_is_managed_when_any_artifact_is_managed() -> None:
assert versions["1.0.0"].managed is True
+def test_version_order_breaks_simple_ties_by_version_string() -> None:
+ items = [("1.0", _NOW, None), ("1.1", _NOW, None), ("0.9", _NOW, 5)]
+
+ assert catalog.version_order(sql.VersionMethod.SIMPLE, items) == ["0.9", "1.1", "1.0"]
+
+
+def test_version_order_is_deterministic_for_colliding_loose_labels() -> None:
+ items = [("1.0", None, None), ("1-0", None, None), ("2.0-SNAPSHOT", None, None)]
+
+ assert catalog.version_order(sql.VersionMethod.SEMVER, items) == ["2.0-SNAPSHOT", "1.0", "1-0"]
+
+
+def test_version_order_is_deterministic_for_equal_pep440_labels() -> None:
+ items = [("1.0.0", None, None), ("1.0", None, None), ("2.0", None, None)]
+
+ assert catalog.version_order(sql.VersionMethod.SEMVER, items) == ["2.0", "1.0.0", "1.0"]
+
+
def test_versioned_projects_use_the_grouped_layout() -> None:
assert catalog._grouped_layout(sql.VersionMethod.SEMVER) is True
assert catalog._grouped_layout(sql.VersionMethod.CALVER) is True
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org