You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tooling.apache.org by ar...@apache.org on 2026/05/29 14:22:08 UTC

(tooling-trusted-releases) branch arm updated (e45b956d -> ee48309f)

This is an automated email from the ASF dual-hosted git repository.

alitheg pushed a change to branch arm
in repository https://gitbox.apache.org/repos/asf/tooling-trusted-releases.git


    from e45b956d #1259 - add default email settings to project policy, pull defaults from policy when showing to user, and allow extra CC/BCC when resolving vote.
     add e9ccebf5 Update dulwich
     new ee48309f #139 - Add export of .asf.yaml block to allow seeding

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 atr/get/projects.py                           | 176 ++++++++++++++++++++++----
 atr/storage/writers/tokens.py                 |  12 +-
 pip-audit.requirements                        |   2 +-
 tests/Dockerfile.e2e                          |   2 +-
 tests/e2e/projects/test_asf_yaml_roundtrip.py | 150 ++++++++++++++++++++++
 tests/unit/test_asf_yaml_export.py            | 134 ++++++++++++++++++++
 uv.lock                                       |  26 ++--
 7 files changed, 454 insertions(+), 48 deletions(-)
 create mode 100644 tests/e2e/projects/test_asf_yaml_roundtrip.py
 create mode 100644 tests/unit/test_asf_yaml_export.py


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org


(tooling-trusted-releases) 01/01: #139 - Add export of .asf.yaml block to allow seeding

Posted by ar...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

alitheg pushed a commit to branch arm
in repository https://gitbox.apache.org/repos/asf/tooling-trusted-releases.git

commit ee48309f313017bca6046fb48e312df1d11100d3
Author: Alastair McFarlane <ar...@apache.org>
AuthorDate: Fri May 29 15:21:46 2026 +0100

    #139 - Add export of .asf.yaml block to allow seeding
---
 atr/get/projects.py                           | 176 ++++++++++++++++++++++----
 atr/storage/writers/tokens.py                 |  12 +-
 tests/Dockerfile.e2e                          |   2 +-
 tests/e2e/projects/test_asf_yaml_roundtrip.py | 150 ++++++++++++++++++++++
 tests/unit/test_asf_yaml_export.py            | 134 ++++++++++++++++++++
 5 files changed, 440 insertions(+), 34 deletions(-)

diff --git a/atr/get/projects.py b/atr/get/projects.py
index 16737dab..3598d493 100644
--- a/atr/get/projects.py
+++ b/atr/get/projects.py
@@ -100,6 +100,59 @@ async def add_project(
     )
 
 
+@get.typed
+async def project_yaml(
+    session: web.Committer, _project_yaml: Literal["project/yaml"], project_key: safe.ProjectKey
+) -> web.WerkzeugResponse | str:
+    """
+    URL: /project/yaml/<project_key>
+    """
+    async with db.session() as data:
+        project = await data.project(
+            key=str(project_key),
+            _committee=True,
+            _release_policy=True,
+        ).demand(base.ASFQuartException(f"Project {project_key} not found", errorcode=404))
+
+    is_committee_member = bool(project.committee and user.is_committee_member(project.committee, session.uid))
+    if not (is_committee_member or session.is_admin):
+        raise base.ASFQuartException("You are not a member of this project's committee", errorcode=403)
+
+    yaml_text = _asf_yaml_export(project)
+
+    page = htm.Block()
+    page.p[htm.a(".atr-back-link", href=util.as_url(view, project_key=project.key))["← Back to project"]]
+    page.h1["Export .asf.yaml"]
+    page.p[
+        "This is the ",
+        htm.code[".asf.yaml"],
+        " project block describing what ATR currently holds for ",
+        project.display_name,
+        ". Add it to your repository's ",
+        htm.code[".asf.yaml"],
+        " to keep this metadata in sync from the repository side.",
+    ]
+
+    card = htm.Block(htm.div, classes=".card.mb-4")
+    card.div(".card-header.bg-light.d-flex.justify-content-between.align-items-center")[
+        htm.h3(".mb-0")[".asf.yaml"],
+        htm.button(
+            ".btn.btn-sm.btn-outline-secondary.atr-copy-btn",
+            type="button",
+            data_clipboard_target="#asf-yaml-output",
+        )[htm.i(".bi.bi-clipboard"), " Copy"],
+    ]
+    card.div(".card-body")[htm.pre("#asf-yaml-output.mb-0")[yaml_text]]
+    page.append(card.collect())
+
+    return await template.blank(
+        title="Export .asf.yaml",
+        description=f"The .asf.yaml project block for {project.display_name}.",
+        content=page.collect(),
+        javascripts=["clipboard-copy"],
+    )
+
+
 @get.typed
 async def projects(session: web.Public, _projects: Literal["projects"]) -> str:
     """
@@ -261,6 +314,65 @@ async def view(
     )
 
 
+def _asf_yaml_export(project: sql.Project) -> str:
+    block: dict[str, object] = {"metadata": _asf_yaml_metadata(project)}
+    policy = _asf_yaml_recipient_blocks(project)
+    if policy:
+        block["policy"] = policy
+    # atr_sync defaults to on, so we emit it as a visible reminder the sync exists
+    block["features"] = {"atr_sync": "true"}
+    return strictyaml.as_document({"project": block}).as_yaml()
+
+
+def _asf_yaml_metadata(project: sql.Project) -> dict[str, object]:
+    metadata: dict[str, object] = {}
+    if project.committee_key:
+        metadata["committee"] = project.committee_key
+    if project.name:
+        metadata["name"] = project.name
+    for field in (
+        "description",
+        "short_description",
+        "homepage",
+        "lifecycle_page",
+        "download_page",
+        "bug_database",
+        "mailing_lists",
+    ):
+        value = getattr(project, field)
+        if value:
+            metadata[field] = value
+    if project.repositories:
+        metadata["repositories"] = list(project.repositories)
+    if project.standards:
+        metadata["standards"] = list(project.standards)
+    # categories and programming_languages are stored comma-joined, so split them back out
+    for field, raw in (("categories", project.categories), ("programming_languages", project.programming_languages)):
+        items = [item.strip() for item in (raw or "").split(",") if item.strip()]
+        if items:
+            metadata[field] = items
+    return metadata
+
+
+def _asf_yaml_recipient_blocks(project: sql.Project) -> dict[str, object]:
+    policy: dict[str, object] = {}
+    for key, action in (
+        ("vote_recipients", sql.RecipientAction.VOTE),
+        ("announce_recipients", sql.RecipientAction.ANNOUNCE),
+    ):
+        to, cc, bcc = project.policy_recipients(action)
+        recipients: dict[str, object] = {}
+        if to:
+            recipients["to"] = to
+        if cc:
+            recipients["cc"] = list(cc)
+        if bcc:
+            recipients["bcc"] = list(bcc)
+        if recipients:
+            policy[key] = recipients
+    return policy
+
+
 def _cycle_display_name(cycle: sql.ProjectCycle) -> str:
     # The "default" cycle is what every project gets when it has no cycle_match
     return "No lifecycle information" if cycle.cycle == "default" else f"Version {cycle.cycle}"
@@ -404,6 +516,34 @@ def _input_with_variables(
     return htm.div[elements]
 
 
+def _recipient_grid_widget(project: sql.Project, action: sql.RecipientAction) -> htm.Element:
+    committee = project.committee
+    committee_key = committee.key if (committee is not None) else str(project.key)
+    is_podling = bool(committee is not None and committee.is_podling)
+    options = util.configurable_recipients(action, committee_key, is_podling=is_podling)
+    stored_to, stored_cc, stored_bcc = project.policy_recipients(action)
+    default_to = stored_to if (stored_to in options) else (options[0] if options else None)
+    return htm.div("#email_to")[
+        render.html_recipients_to_radios(options, default_to=default_to),
+        htpy.details(".mt-2")[
+            htpy.summary["Select CC and BCC recipients"],
+            render.html_recipients_cc_bcc_table(options, selected_cc=set(stored_cc), selected_bcc=set(stored_bcc)),
+        ],
+    ]
+
+
+def _recipient_summary(to: str, cc: list[str], bcc: list[str]) -> htm.Element | str:
+    if not to:
+        return "Not set"
+    extras = []
+    if cc:
+        extras.append(f"{len(cc)} CC")
+    if bcc:
+        extras.append(f"{len(bcc)} BCC")
+    suffix = f" (+{', '.join(extras)})" if extras else ""
+    return htm.span[htm.a(href=f"mailto:{to}")[to], suffix]
+
+
 def _release_url(project: sql.Project, release: sql.Release) -> str:
     project_key = str(project.key)
     version_key = str(release.version)
@@ -439,6 +579,14 @@ async def _render_actions_card(
                         )["Create a sibling project"]
                     ]
                 )
+                body.append(
+                    htm.p[
+                        htm.a(
+                            ".btn.btn-sm.btn-outline-secondary",
+                            href=util.as_url(project_yaml, project_key=project.key),
+                        )["Export .asf.yaml"]
+                    ]
+                )
     return card.collect()
 
 
@@ -1108,34 +1256,6 @@ async def _render_vote_form(project: sql.Project) -> htm.Element:
     return card.collect()
 
 
-def _recipient_summary(to: str, cc: list[str], bcc: list[str]) -> htm.Element | str:
-    if not to:
-        return "Not set"
-    extras = []
-    if cc:
-        extras.append(f"{len(cc)} CC")
-    if bcc:
-        extras.append(f"{len(bcc)} BCC")
-    suffix = f" (+{', '.join(extras)})" if extras else ""
-    return htm.span[htm.a(href=f"mailto:{to}")[to], suffix]
-
-
-def _recipient_grid_widget(project: sql.Project, action: sql.RecipientAction) -> htm.Element:
-    committee = project.committee
-    committee_key = committee.key if (committee is not None) else str(project.key)
-    is_podling = bool(committee is not None and committee.is_podling)
-    options = util.configurable_recipients(action, committee_key, is_podling=is_podling)
-    stored_to, stored_cc, stored_bcc = project.policy_recipients(action)
-    default_to = stored_to if (stored_to in options) else (options[0] if options else None)
-    return htm.div("#email_to")[
-        render.html_recipients_to_radios(options, default_to=default_to),
-        htpy.details(".mt-2")[
-            htpy.summary["Select CC and BCC recipients"],
-            render.html_recipients_cc_bcc_table(options, selected_cc=set(stored_cc), selected_bcc=set(stored_bcc)),
-        ],
-    ]
-
-
 def _textarea_with_variables(
     field_name: str,
     default_value: str,
diff --git a/atr/storage/writers/tokens.py b/atr/storage/writers/tokens.py
index 06f07cdd..62f334fd 100644
--- a/atr/storage/writers/tokens.py
+++ b/atr/storage/writers/tokens.py
@@ -24,6 +24,7 @@ import hashlib
 
 import sqlmodel
 
+import atr.config as config
 import atr.db as db
 import atr.jwtoken as jwtoken
 import atr.ldap as ldap
@@ -127,11 +128,12 @@ class FoundationCommitter(GeneralPublic):
             )
             raise storage.AccessError("Authentication failed", status=401)
 
-        # Verify account still exists in LDAP
-        account_details = await ldap.account_lookup(self.__asf_uid)
-        if (account_details is None) or ldap.is_banned(account_details):
-            log.auth_failure("jwt_issuance", "account_deleted_or_banned", self.__asf_uid)
-            raise storage.AccessError("Authentication failed", status=401)
+        # Verify account still exists in LDAP. Test mode doesn't have LDAP wired up so we skip it
+        if not config.is_test_mode():
+            account_details = await ldap.account_lookup(self.__asf_uid)
+            if (account_details is None) or ldap.is_banned(account_details):
+                log.auth_failure("jwt_issuance", "account_deleted_or_banned", self.__asf_uid)
+                raise storage.AccessError("Authentication failed", status=401)
 
         issued_jwt = jwtoken.issue(self.__asf_uid, pat_hash=pat_hash)
         log.auth_event("jwt_issued", self.__asf_uid, pat_hash=pat_hash)
diff --git a/tests/Dockerfile.e2e b/tests/Dockerfile.e2e
index 90bd9b5c..6b6730ef 100644
--- a/tests/Dockerfile.e2e
+++ b/tests/Dockerfile.e2e
@@ -1,6 +1,6 @@
 FROM mcr.microsoft.com/playwright/python:v1.58.0-noble
 
-RUN pip3 install --no-cache-dir --break-system-packages pytest pytest-playwright
+RUN pip3 install --no-cache-dir --break-system-packages pytest pytest-playwright playwright==1.58.0 strictyaml
 
 COPY . /run/tests/e2e
 WORKDIR /run/tests
diff --git a/tests/e2e/projects/test_asf_yaml_roundtrip.py b/tests/e2e/projects/test_asf_yaml_roundtrip.py
new file mode 100644
index 00000000..47e72f92
--- /dev/null
+++ b/tests/e2e/projects/test_asf_yaml_roundtrip.py
@@ -0,0 +1,150 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from __future__ import annotations
+
+import json
+import os
+from typing import TYPE_CHECKING, Any, Final
+
+import e2e.helpers as helpers
+import pytest
+import strictyaml
+from playwright.sync_api import expect
+
+if TYPE_CHECKING:
+    from collections.abc import Generator
+
+    from playwright.sync_api import APIRequestContext, Browser, BrowserContext, Page
+
+_BASE_URL: Final[str] = os.environ.get("ATR_BASE_URL", "https://localhost.apache.org:8080")
+
+PROJECT_KEY: Final[str] = "test-asfyaml"
+EXPORT_URL: Final[str] = f"/project/yaml/{PROJECT_KEY}"
+
+# The .asf.yaml project block we import, then expect to get back unchanged on export.
+# It mirrors the inbound schema from infrastructure-asfyaml exactly, so a faithful
+# round-trip should reproduce it byte-for-byte once parsed.
+SOURCE_YAML: Final[str] = """\
+project:
+  metadata:
+    committee: test
+    name: Apache Test Asfyaml
+    short_description: A round-trip fixture project.
+    homepage: https://test.apache.org/
+    lifecycle_page: https://test.apache.org/lifecycle
+    download_page: https://test.apache.org/download
+    bug_database: https://issues.apache.org/jira/browse/TESTASFYAML
+    mailing_lists: https://test.apache.org/mail
+    repositories:
+    - https://github.com/apache/test-asfyaml.git
+    standards:
+    - https://www.rfc-editor.org/rfc/rfc9999
+    categories:
+    - build-management
+    - testing
+    programming_languages:
+    - python
+  policy:
+    vote_recipients:
+      to: private@test.apache.org
+      cc:
+      - dev@test.apache.org
+    announce_recipients:
+      to: announce@test.apache.org
+  features:
+    atr_sync: true
+"""
+
+
+@pytest.fixture
+def export_page(roundtrip_context: BrowserContext) -> Generator[Page]:
+    page = roundtrip_context.new_page()
+    helpers.visit(page, EXPORT_URL)
+    yield page
+    page.close()
+
+
+@pytest.fixture(scope="module")
+def roundtrip_context(browser: Browser) -> Generator[BrowserContext]:
+    context = browser.new_context(ignore_https_errors=True)
+    page = context.new_page()
+    helpers.log_in(page)
+    _import_project(page.request, _mint_pat_and_jwt(page))
+    page.close()
+    yield context
+    cleanup = context.new_page()
+    helpers.log_in(cleanup)
+    _delete_project(cleanup)
+    cleanup.close()
+    context.close()
+
+
+def test_export_round_trips_imported_yaml(export_page: Page) -> None:
+    """Exporting a project reproduces the .asf.yaml block that was imported into it."""
+    exported = export_page.locator("#asf-yaml-output").inner_text()
+    assert strictyaml.load(exported).data == strictyaml.load(SOURCE_YAML).data
+
+
+def _delete_project(page: Page) -> None:
+    # A project with no releases offers a "Delete project" button in its Actions card.
+    helpers.visit(page, f"/projects/{PROJECT_KEY}")
+    button = page.get_by_role("button", name="Delete project")
+    if button.count() == 0:
+        return
+    page.once("dialog", lambda dialog: dialog.accept())
+    button.click()
+    page.wait_for_load_state()
+
+
+def _import_payload() -> dict[str, Any]:
+    # Replays what infrastructure-asfyaml does: committee is authored inside the
+    # metadata block but the API expects it at the top level, so pop it out.
+    block = strictyaml.load(SOURCE_YAML).data["project"]
+    metadata = dict(block["metadata"])
+    committee_key = metadata.pop("committee")
+    payload: dict[str, Any] = {
+        "project_key": PROJECT_KEY,
+        "committee_key": committee_key,
+        "project": metadata,
+    }
+    if "policy" in block:
+        payload["policy"] = block["policy"]
+    return payload
+
+
+def _import_project(request: APIRequestContext, jwt: str) -> None:
+    response = request.post(
+        f"{_BASE_URL}/api/project/config",
+        headers={"Authorization": f"Bearer {jwt}", "Content-Type": "application/json"},
+        data=json.dumps(_import_payload()),
+    )
+    if not response.ok:
+        raise RuntimeError(f"Import via /api/project/config failed ({response.status}): {response.text()}")
+
+
+def _mint_pat_and_jwt(page: Page) -> str:
+    helpers.visit(page, "/tokens")
+    page.locator('input[name="label"]').fill("e2e-asfyaml-roundtrip")
+    page.get_by_role("button", name="Generate token").click()
+    page.wait_for_load_state()
+    pat = page.locator(".flash-success code").first.inner_text().strip()
+
+    page.locator('#issue-jwt-form input[name="pat"]').fill(pat)
+    page.get_by_role("button", name="Generate JWT").click()
+    expect(page.locator("#jwt-container")).to_be_visible()
+    return page.locator("#jwt-output").inner_text().strip()
diff --git a/tests/unit/test_asf_yaml_export.py b/tests/unit/test_asf_yaml_export.py
new file mode 100644
index 00000000..b356c519
--- /dev/null
+++ b/tests/unit/test_asf_yaml_export.py
@@ -0,0 +1,134 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import strictyaml
+
+import atr.get.projects as projects
+import atr.models.sql as sql
+
+
+class _FakeProject:
+    def __init__(self, recipients=None, **fields):
+        defaults = {
+            "committee_key": None,
+            "name": None,
+            "description": None,
+            "short_description": None,
+            "homepage": None,
+            "lifecycle_page": None,
+            "download_page": None,
+            "bug_database": None,
+            "mailing_lists": None,
+            "repositories": [],
+            "standards": [],
+            "categories": None,
+            "programming_languages": None,
+        }
+        defaults.update(fields)
+        for key, value in defaults.items():
+            setattr(self, key, value)
+        self._recipients = recipients or {}
+
+    def policy_recipients(self, action):
+        entry = self._recipients.get(action, {})
+        return entry.get("to", ""), list(entry.get("cc", [])), list(entry.get("bcc", []))
+
+
+def test_export_includes_only_recipient_keys_that_are_set() -> None:
+    project = _FakeProject(
+        committee_key="tooling",
+        name="Apache Example",
+        recipients={sql.RecipientAction.VOTE: {"to": "private@example.apache.org"}},
+    )
+
+    policy = strictyaml.load(projects._asf_yaml_export(project)).data["project"]["policy"]
+
+    assert policy == {"vote_recipients": {"to": "private@example.apache.org"}}
+
+
+def test_export_minimal_project_omits_empty_blocks() -> None:
+    project = _FakeProject(committee_key="tooling", name="Apache Trusted Releases")
+
+    loaded = strictyaml.load(projects._asf_yaml_export(project)).data
+
+    assert loaded == {
+        "project": {
+            "metadata": {"committee": "tooling", "name": "Apache Trusted Releases"},
+            "features": {"atr_sync": "true"},
+        }
+    }
+
+
+def test_export_reproduces_every_field() -> None:
+    project = _FakeProject(
+        committee_key="tooling",
+        name="Apache Trusted Releases",
+        short_description="A platform for making official ASF software releases.",
+        homepage="https://tooling.apache.org/trusted-releases.html",
+        lifecycle_page="https://tooling.apache.org/trusted-releases.html",
+        download_page="https://github.com/apache/tooling-trusted-releases",
+        bug_database="https://github.com/apache/tooling-trusted-releases/issues",
+        mailing_lists="https://tooling.apache.org/volunteer.html",
+        repositories=["git+ssh://git@github.com:apache/tooling-trusted-releases.git"],
+        standards=["https://owasp.org/www-project-application-security-verification-standard/"],
+        categories="build-management",
+        programming_languages="python",
+        recipients={
+            sql.RecipientAction.VOTE: {"to": "private@tooling.apache.org", "cc": ["dev@tooling.apache.org"]},
+            sql.RecipientAction.ANNOUNCE: {"to": "announce@apache.org"},
+        },
+    )
+
+    loaded = strictyaml.load(projects._asf_yaml_export(project)).data
+
+    assert loaded == {
+        "project": {
+            "metadata": {
+                "committee": "tooling",
+                "name": "Apache Trusted Releases",
+                "short_description": "A platform for making official ASF software releases.",
+                "homepage": "https://tooling.apache.org/trusted-releases.html",
+                "lifecycle_page": "https://tooling.apache.org/trusted-releases.html",
+                "download_page": "https://github.com/apache/tooling-trusted-releases",
+                "bug_database": "https://github.com/apache/tooling-trusted-releases/issues",
+                "mailing_lists": "https://tooling.apache.org/volunteer.html",
+                "repositories": ["git+ssh://git@github.com:apache/tooling-trusted-releases.git"],
+                "standards": ["https://owasp.org/www-project-application-security-verification-standard/"],
+                "categories": ["build-management"],
+                "programming_languages": ["python"],
+            },
+            "policy": {
+                "vote_recipients": {"to": "private@tooling.apache.org", "cc": ["dev@tooling.apache.org"]},
+                "announce_recipients": {"to": "announce@apache.org"},
+            },
+            "features": {"atr_sync": "true"},
+        }
+    }
+
+
+def test_export_splits_comma_joined_columns_into_sequences() -> None:
+    project = _FakeProject(
+        committee_key="tooling",
+        name="Apache Example",
+        categories="data, storage",
+        programming_languages="c,python",
+    )
+
+    metadata = strictyaml.load(projects._asf_yaml_export(project)).data["project"]["metadata"]
+
+    assert metadata["categories"] == ["data", "storage"]
+    assert metadata["programming_languages"] == ["c", "python"]


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org