You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tooling.apache.org by ar...@apache.org on 2026/01/12 14:25:25 UTC
(tooling-trusted-releases) 01/01: Rename GHA task to be more specific, and work in extra required parameters. UI work to trigger distributions and endpoint to register an SSH key
This is an automated email from the ASF dual-hosted git repository.
arm pushed a commit to branch gha-distributions
in repository https://gitbox.apache.org/repos/asf/tooling-trusted-releases.git
commit cabce79158ee5a551c5a7a032239036b0da9534e
Author: Alastair McFarlane <ar...@apache.org>
AuthorDate: Mon Jan 12 14:24:37 2026 +0000
Rename GHA task to be more specific, and work in extra required parameters. UI work to trigger distributions and endpoint to register an SSH key
---
atr/api/__init__.py | 30 ++++++++-
atr/db/interaction.py | 18 +++++
atr/form.py | 4 ++
atr/get/distribution.py | 85 ++++++++++++++++++-----
atr/get/finish.py | 126 ++++++++++++++++++++++++++++-------
atr/models/api.py | 15 +++++
atr/models/results.py | 6 +-
atr/models/sql.py | 10 ++-
atr/post/distribution.py | 107 ++++++++++++++++++++++++++---
atr/server.py | 3 +-
atr/shared/distribution.py | 13 ++--
atr/storage/writers/distributions.py | 50 +++++++++++++-
atr/tasks/__init__.py | 2 +-
atr/tasks/gha.py | 96 +++++++++++++++++++++-----
atr/templates/check-selected.html | 12 +++-
15 files changed, 492 insertions(+), 85 deletions(-)
diff --git a/atr/api/__init__.py b/atr/api/__init__.py
index 06c932e..e543b43 100644
--- a/atr/api/__init__.py
+++ b/atr/api/__init__.py
@@ -251,6 +251,32 @@ async def committees_list() -> DictResponse:
).model_dump(), 200
+@api.route("/distribute/ssh/register", methods=["POST"])
+@quart_schema.validate_request(models.api.DistributeSshRegisterArgs)
+async def distribute_ssh_register(data: models.api.DistributeSshRegisterArgs) -> DictResponse:
+ """
+ Register an SSH key sent with a corroborating Trusted Publisher JWT,
+ validating the requested version is in the correct phase.
+ """
+ payload, asf_uid, project = await interaction.trusted_jwt_for_version(
+ data.publisher, data.jwt, interaction.TrustedProjectPhase(data.phase), data.version
+ )
+ async with storage.write_as_committee_member(util.unwrap(project.committee).name, asf_uid) as wacm:
+ fingerprint, expires = await wacm.ssh.add_workflow_key(
+ payload["actor"],
+ payload["actor_id"],
+ project.name,
+ data.ssh_key,
+ )
+
+ return models.api.DistributeSshRegisterResults(
+ endpoint="/distribute/ssh/register",
+ fingerprint=fingerprint,
+ project=project.name,
+ expires=expires,
+ ).model_dump(), 200
+
+
@api.route("/distribution/record", methods=["POST"])
@jwtoken.require
@quart_schema.security_scheme([{"BearerAuth": []}])
@@ -279,7 +305,7 @@ async def distribution_record(data: models.api.DistributionRecordArgs) -> DictRe
async with storage.write(asf_uid) as write:
wacm = write.as_committee_member(release.committee.name)
await wacm.distributions.record_from_data(
- release,
+ release.name,
data.staging,
dd,
)
@@ -656,7 +682,7 @@ async def publisher_distribution_record(data: models.api.PublisherDistributionRe
async with storage.write(asf_uid) as write:
wacm = write.as_committee_member(release.committee.name)
await wacm.distributions.record_from_data(
- release,
+ release.name,
data.staging,
dd,
)
diff --git a/atr/db/interaction.py b/atr/db/interaction.py
index f8026f8..1bd097c 100644
--- a/atr/db/interaction.py
+++ b/atr/db/interaction.py
@@ -179,6 +179,24 @@ async def trusted_jwt(publisher: str, jwt: str, phase: TrustedProjectPhase) -> t
return payload, asf_uid, project
+async def trusted_jwt_for_version(
+ publisher: str, jwt: str, phase: TrustedProjectPhase, version_name: str
+) -> tuple[dict[str, Any], str, sql.Project]:
+ payload, asf_uid, project = await trusted_jwt(publisher, jwt, phase)
+ async with db.session() as db_data:
+ release = await db_data.release(project_name=project.name, version=version_name).get()
+ if not release:
+ raise InteractionError(f"Release {version} does not exist in project {project.name}")
+ if phase == TrustedProjectPhase.COMPOSE and release.phase != sql.ReleasePhase.RELEASE_CANDIDATE_DRAFT:
+ raise InteractionError(f"Release {version} is not in compose phase")
+ if phase == TrustedProjectPhase.VOTE and release.phase != sql.ReleasePhase.RELEASE_CANDIDATE:
+ raise InteractionError(f"Release {version} is not in vote phase")
+ if phase == TrustedProjectPhase.FINISH and release.phase != sql.ReleasePhase.RELEASE_PREVIEW:
+ raise InteractionError(f"Release {version} is not in finish phase")
+
+ return payload, asf_uid, project
+
+
async def has_failing_checks(release: sql.Release, revision_number: str, caller_data: db.Session | None = None) -> bool:
async with db.ensure_session(caller_data) as data:
query = (
diff --git a/atr/form.py b/atr/form.py
index 6f40775..909b0e6 100644
--- a/atr/form.py
+++ b/atr/form.py
@@ -517,6 +517,10 @@ Email = pydantic.EmailStr
class Enum[EnumType: enum.Enum]:
+ # These exist for type checkers - at runtime, the actual type is the enum
+ name: str
+ value: str | int
+
@staticmethod
def __class_getitem__(enum_class: type[EnumType]):
def validator(v: Any) -> EnumType:
diff --git a/atr/get/distribution.py b/atr/get/distribution.py
index 4f166a9..9a284d7 100644
--- a/atr/get/distribution.py
+++ b/atr/get/distribution.py
@@ -29,30 +29,30 @@ import atr.util as util
import atr.web as web
-@get.committer("/distributions/list/<project>/<version>")
-async def list_get(session: web.Committer, project: str, version: str) -> str:
+@get.committer("/distributions/list/<project_name>/<version_name>")
+async def list_get(session: web.Committer, project_name: str, version_name: str) -> str:
async with db.session() as data:
distributions = await data.distribution(
- release_name=sql.release_name(project, version),
+ release_name=sql.release_name(project_name, version_name),
).all()
block = htm.Block()
- release = await shared.distribution.release_validated(project, version, staging=None)
+ release = await shared.distribution.release_validated(project_name, version_name, staging=None)
staging = release.phase == sql.ReleasePhase.RELEASE_CANDIDATE_DRAFT
- render.html_nav_phase(block, project, version, staging)
+ render.html_nav_phase(block, project_name, version_name, staging)
record_a_distribution = htm.a(
".btn.btn-primary",
href=util.as_url(
- stage if staging else record,
- project=project,
- version=version,
+ stage_record if staging else record,
+ project=project_name,
+ version=version_name,
),
)["Record a distribution"]
# Distribution list for project-version
- block.h1["Distribution list for ", htm.em[f"{project}-{version}"]]
+ block.h1["Distribution list for ", htm.em[f"{project_name}-{version_name}"]]
if not distributions:
block.p["No distributions found."]
block.p[record_a_distribution]
@@ -89,7 +89,7 @@ async def list_get(session: web.Committer, project: str, version: str) -> str:
delete_form = form.render(
model_cls=shared.distribution.DeleteForm,
- action=util.as_url(post.distribution.delete, project=project, version=version),
+ action=util.as_url(post.distribution.delete, project=project_name, version=version_name),
form_classes=".d-inline-block.m-0",
submit_classes="btn-danger btn-sm",
submit_label="Delete",
@@ -105,20 +105,71 @@ async def list_get(session: web.Committer, project: str, version: str) -> str:
)
block.append(htm.div(".mb-3")[delete_form])
- title = f"Distribution list for {project} {version}"
+ title = f"Distribution list for {project_name} {version_name}"
return await template.blank(title, content=block.collect())
+@get.committer("/distribution/automate/<project>/<version>")
+async def automate(session: web.Committer, project: str, version: str) -> str:
+ return await _automate_form_page(project, version, staging=False)
+
+
+@get.committer("/distribution/stage/automate/<project>/<version>")
+async def stage_automate(session: web.Committer, project: str, version: str) -> str:
+ return await _automate_form_page(project, version, staging=True)
+
+
@get.committer("/distribution/record/<project>/<version>")
async def record(session: web.Committer, project: str, version: str) -> str:
return await _record_form_page(project, version, staging=False)
-@get.committer("/distribution/stage/<project>/<version>")
-async def stage(session: web.Committer, project: str, version: str) -> str:
+@get.committer("/distribution/stage/record/<project>/<version>")
+async def stage_record(session: web.Committer, project: str, version: str) -> str:
return await _record_form_page(project, version, staging=True)
+async def _automate_form_page(project: str, version: str, staging: bool) -> str:
+ """Helper to render the distribution automation form page."""
+ await shared.distribution.release_validated(project, version, staging=staging)
+
+ block = htm.Block()
+ render.html_nav_phase(block, project, version, staging=staging)
+
+ title = "Create a staging distribution" if staging else "Create a distribution"
+ block.h1[title]
+
+ block.p[
+ "Create a distribution of ",
+ htm.strong[f"{project}-{version}"],
+ " using the form below.",
+ ]
+ block.p[
+ "You can also ",
+ htm.a(href=util.as_url(list_get, project_name=project, version_name=version))["view the distribution list"],
+ ".",
+ ]
+
+ # Determine the action based on staging
+ action = (
+ util.as_url(post.distribution.stage_automate_selected, project=project, version=version)
+ if staging
+ else util.as_url(post.distribution.automate_selected, project=project, version=version)
+ )
+
+ # TODO: Reuse the same form for now - maybe we can combine this and the function below adding an automate=True arg
+ # Render the distribution form
+ form_html = form.render(
+ model_cls=shared.distribution.DistributeForm,
+ submit_label="Distribute",
+ action=action,
+ defaults={"package": project, "version": version},
+ )
+ block.append(form_html)
+
+ return await template.blank(title, content=block.collect())
+
+
async def _record_form_page(project: str, version: str, staging: bool) -> str:
"""Helper to render the distribution recording form page."""
await shared.distribution.release_validated(project, version, staging=staging)
@@ -126,23 +177,23 @@ async def _record_form_page(project: str, version: str, staging: bool) -> str:
block = htm.Block()
render.html_nav_phase(block, project, version, staging=staging)
- title = "Record a staging distribution" if staging else "Record a manual distribution"
+ title = "Record a manual staging distribution" if staging else "Record a manual distribution"
block.h1[title]
block.p[
- "Record a distribution of ",
+ "Record a manual distribution of ",
htm.strong[f"{project}-{version}"],
" using the form below.",
]
block.p[
"You can also ",
- htm.a(href=util.as_url(list_get, project=project, version=version))["view the distribution list"],
+ htm.a(href=util.as_url(list_get, project_name=project, version_name=version))["view the distribution list"],
".",
]
# Determine the action based on staging
action = (
- util.as_url(post.distribution.stage_selected, project=project, version=version)
+ util.as_url(post.distribution.stage_record_selected, project=project, version=version)
if staging
else util.as_url(post.distribution.record_selected, project=project, version=version)
)
diff --git a/atr/get/finish.py b/atr/get/finish.py
index 921bf83..3526853 100644
--- a/atr/get/finish.py
+++ b/atr/get/finish.py
@@ -19,6 +19,7 @@
import dataclasses
import json
import pathlib
+from collections.abc import Sequence
import aiofiles.os
import asfquart.base as base
@@ -42,6 +43,7 @@ import atr.mapping as mapping
import atr.models.sql as sql
import atr.render as render
import atr.shared as shared
+import atr.tasks.gha as gha
import atr.template as template
import atr.util as util
import atr.web as web
@@ -60,13 +62,9 @@ async def selected(
) -> tuple[web.QuartResponse, int] | web.WerkzeugResponse | str:
"""Finish a release preview."""
try:
- (
- release,
- source_files_rel,
- target_dirs,
- deletable_dirs,
- rc_analysis,
- ) = await _get_page_data(project_name, version_name)
+ (release, source_files_rel, target_dirs, deletable_dirs, rc_analysis, tasks) = await _get_page_data(
+ project_name, version_name
+ )
except ValueError:
async with db.session() as data:
release_fallback = await data.release(
@@ -89,6 +87,7 @@ async def selected(
target_dirs=target_dirs,
deletable_dirs=deletable_dirs,
rc_analysis=rc_analysis,
+ distribution_tasks=tasks,
)
@@ -134,14 +133,31 @@ async def _deletable_choices(
async def _get_page_data(
project_name: str, version_name: str
-) -> tuple[sql.Release, list[pathlib.Path], set[pathlib.Path], list[tuple[str, str]], RCTagAnalysisResult]:
+) -> tuple[
+ sql.Release, list[pathlib.Path], set[pathlib.Path], list[tuple[str, str]], RCTagAnalysisResult, Sequence[sql.Task]
+]:
"""Get all the data needed to render the finish page."""
async with db.session() as data:
+ via = sql.validate_instrumented_attribute
release = await data.release(
project_name=project_name,
version=version_name,
_committee=True,
).demand(base.ASFQuartException("Release does not exist", errorcode=404))
+ tasks = [
+ t
+ for t in (
+ await data.task(
+ project_name=project_name,
+ version_name=version_name,
+ revision_number=release.latest_revision_number,
+ task_type=sql.TaskType.DISTRIBUTION_WORKFLOW,
+ )
+ .order_by(sql.sqlmodel.desc(via(sql.Task.started)))
+ .all()
+ )
+ if t.status in [sql.TaskStatus.QUEUED, sql.TaskStatus.ACTIVE, sql.TaskStatus.FAILED]
+ ]
if release.phase != sql.ReleasePhase.RELEASE_PREVIEW:
raise ValueError("Release is not in preview phase")
@@ -151,7 +167,7 @@ async def _get_page_data(
deletable_dirs = await _deletable_choices(latest_revision_dir, target_dirs)
rc_analysis_result = await _analyse_rc_tags(latest_revision_dir)
- return release, source_files_rel, target_dirs, deletable_dirs, rc_analysis_result
+ return release, source_files_rel, target_dirs, deletable_dirs, rc_analysis_result, tasks
def _render_delete_directory_form(deletable_dirs: list[tuple[str, str]]) -> htm.Element:
@@ -247,6 +263,7 @@ async def _render_page(
target_dirs: set,
deletable_dirs: list[tuple[str, str]],
rc_analysis: RCTagAnalysisResult,
+ distribution_tasks: Sequence[sql.Task],
) -> str:
"""Render the finish page using htm.py."""
page = htm.Block()
@@ -275,8 +292,11 @@ async def _render_page(
"such as Maven Central, PyPI, or Docker Hub."
]
- # TODO alert
- page.append(_render_todo_alert(release))
+ if len(distribution_tasks) > 0:
+ page.append(_render_distribution_tasks(release, distribution_tasks))
+
+ page.append(_render_dist_warning())
+ page.append(_render_distribution_buttons(release))
# Move files section
page.append(_render_move_section(max_files_to_show=10))
@@ -401,7 +421,7 @@ def _render_release_card(release: sql.Release) -> htm.Element:
version_name=release.version,
),
)[
- htpy.i(".bi.bi-download"),
+ htm.icon("download"),
" Download all files",
],
htm.a(
@@ -413,7 +433,7 @@ def _render_release_card(release: sql.Release) -> htm.Element:
version_name=release.version,
),
)[
- htpy.i(".bi.bi-archive"),
+ htm.icon("archive"),
" Show files",
],
htm.a(
@@ -425,7 +445,7 @@ def _render_release_card(release: sql.Release) -> htm.Element:
version_name=release.version,
),
)[
- htpy.i(".bi.bi-clock-history"),
+ htm.icon("clock-history"),
" Show revisions",
],
htm.a(
@@ -437,7 +457,7 @@ def _render_release_card(release: sql.Release) -> htm.Element:
version_name=release.version,
),
)[
- htpy.i(".bi.bi-check-circle"),
+ htm.icon("check-circle"),
" Announce and distribute",
],
],
@@ -446,25 +466,83 @@ def _render_release_card(release: sql.Release) -> htm.Element:
return card
-def _render_todo_alert(release: sql.Release) -> htm.Element:
- """Render the TODO alert about distribution tools."""
- return htm.div(".alert.alert-warning.mb-4", role="alert")[
- htm.p(".fw-semibold.mb-1")["TODO"],
+def _render_distribution_buttons(release: sql.Release) -> htm.Element:
+ """Render the distribution tool buttons."""
+ return htm.div()[
htm.p(".mb-1")[
- "We plan to add tools to help release managers to distribute release artifacts on distribution networks. "
- "Currently you must do this manually. Once you've distributed your release artifacts, you can ",
htm.a(
+ ".btn.btn-primary.me-2",
+ href=util.as_url(
+ distribution.automate,
+ project=release.project.name,
+ version=release.version,
+ ),
+ )["Distribute"],
+ htm.a(
+ ".btn.btn-secondary.me-2",
href=util.as_url(
distribution.record,
project=release.project.name,
version=release.version,
- )
- )["record them on the ATR"],
- ".",
+ ),
+ )["Record a manual distribution"],
+ ],
+ ]
+
+
+def _render_distribution_tasks(release: sql.Release, tasks: Sequence[sql.Task]) -> htm.Element:
+ """Render current and failed distribution tasks."""
+ failed_tasks = [t for t in tasks if t.status == sql.TaskStatus.FAILED]
+ in_progress_tasks = [t for t in tasks if t.status in [sql.TaskStatus.QUEUED, sql.TaskStatus.ACTIVE]]
+
+ block = htm.Block()
+
+ if len(failed_tasks) > 0:
+ summary = f"{len(failed_tasks)} distribution{'s' if len(failed_tasks) > 1 else ''} failed for this release"
+ block.append(
+ htm.div(".alert.alert-danger.mb-3")[
+ htm.h3["Failed distributions"],
+ htm.details[
+ htm.summary[summary],
+ htm.div[*[_render_task(f) for f in failed_tasks]],
+ ],
+ ]
+ )
+ if len(in_progress_tasks) > 0:
+ block.append(
+ htm.div(".alert.alert-info.mb-3")[
+ htm.h3["In-progress distributions"],
+ htm.p["One or more automatic distributions are still in-progress:"],
+ *[_render_task(f) for f in in_progress_tasks],
+ htm.button(
+ ".btn.btn-success.me-2",
+ {"onclick": "window.location.reload()"},
+ )["Refresh"],
+ ]
+ )
+ return block.collect()
+
+
+def _render_dist_warning() -> htm.Element:
+ """Render the alert about distribution tools."""
+ return htm.div(".alert.alert-warning.mb-4", role="alert")[
+ htm.p(".fw-semibold.mb-1")["NOTE:"],
+ htm.p(".mb-1")[
+ "Tools to distribute automatically are still being developed, "
+ "you must do this manually at present. Please use the manual record function below to do so.",
],
]
+def _render_task(task: sql.Task) -> htm.Element:
+ """Render a distribution task's details."""
+ args: gha.DistributionWorkflow = gha.DistributionWorkflow.model_validate(task.task_args)
+ status = task.status.value
+ return htm.p[
+ f"{args.platform} ({args.package} {args.version}): {task.error if task.error else status.capitalize()}"
+ ]
+
+
async def _sources_and_targets(latest_revision_dir: pathlib.Path) -> tuple[list[pathlib.Path], set[pathlib.Path]]:
source_items_rel: list[pathlib.Path] = []
target_dirs: set[pathlib.Path] = {pathlib.Path(".")}
diff --git a/atr/models/api.py b/atr/models/api.py
index d06c5d3..6df6a14 100644
--- a/atr/models/api.py
+++ b/atr/models/api.py
@@ -67,6 +67,21 @@ class CommitteesListResults(schema.Strict):
committees: Sequence[sql.Committee]
+class DistributeSshRegisterArgs(schema.Strict):
+ publisher: str = schema.example("user")
+ jwt: str = schema.example("eyJhbGciOiJIUzI1[...]mMjLiuyu5CSpyHI=")
+ ssh_key: str = schema.example("ssh-ed25519 AAAAC3NzaC1lZDI1NTEgH5C9okWi0dh25AAAAIOMqqnkVzrm0SdG6UOoqKLsabl9GKJl")
+ phase: str = schema.Field(strict=False, default="compose", json_schema_extra={"examples": ["compose", "finish"]})
+ version: str = schema.example("0.0.1")
+
+
+class DistributeSshRegisterResults(schema.Strict):
+ endpoint: Literal["/distribute/ssh/register"] = schema.alias("endpoint")
+ fingerprint: str = schema.example("SHA256:0123456789abcdef0123456789abcdef01234567")
+ project: str = schema.example("example")
+ expires: int = schema.example(1713547200)
+
+
class DistributionRecordArgs(schema.Strict):
project: str = schema.example("example")
version: str = schema.example("0.0.1")
diff --git a/atr/models/results.py b/atr/models/results.py
index 713291d..5dc1ce2 100644
--- a/atr/models/results.py
+++ b/atr/models/results.py
@@ -24,10 +24,10 @@ import atr.sbom.models.osv as osv
from . import schema
-class GithubActionsWorkflow(schema.Strict):
+class DistributionWorkflow(schema.Strict):
"""Result of the task to run a Github workflow."""
- kind: Literal["github_actions_workflow"] = schema.Field(alias="kind")
+ kind: Literal["distribution_workflow"] = schema.Field(alias="kind")
name: str = schema.description("The name of the action being performed")
run_id: int = schema.description("The ID of the workflow run")
url: str = schema.description("The URL of the workflow run")
@@ -193,7 +193,7 @@ class MetadataUpdate(schema.Strict):
Results = Annotated[
- GithubActionsWorkflow
+ DistributionWorkflow
| HashingCheck
| MessageSend
| MetadataUpdate
diff --git a/atr/models/sql.py b/atr/models/sql.py
index 8957123..8938aeb 100644
--- a/atr/models/sql.py
+++ b/atr/models/sql.py
@@ -53,6 +53,7 @@ sqlmodel.SQLModel.metadata = sqlalchemy.MetaData(
@dataclasses.dataclass(frozen=True)
class DistributionPlatformValue:
name: str
+ gh_slug: str
template_url: str
template_staging_url: str | None = None
requires_owner_namespace: bool = False
@@ -95,12 +96,14 @@ class CheckResultStatusIgnore(str, enum.Enum):
class DistributionPlatform(enum.Enum):
ARTIFACT_HUB = DistributionPlatformValue(
name="Artifact Hub",
+ gh_slug="artifacthub",
template_url="https://artifacthub.io/api/v1/packages/helm/{owner_namespace}/{package}/{version}",
template_staging_url="https://staging.artifacthub.io/api/v1/packages/helm/{owner_namespace}/{package}/{version}",
requires_owner_namespace=True,
)
DOCKER_HUB = DistributionPlatformValue(
name="Docker Hub",
+ gh_slug="dockerhub",
template_url="https://hub.docker.com/v2/namespaces/{owner_namespace}/repositories/{package}/tags/{version}",
# TODO: Need to use staging tags?
# template_staging_url="https://hub.docker.com/v2/namespaces/{owner_namespace}/repositories/{package}/tags/{version}",
@@ -108,6 +111,7 @@ class DistributionPlatform(enum.Enum):
)
# GITHUB = DistributionPlatformValue(
# name="GitHub",
+ # gh_slug="github",
# template_url="https://api.github.com/repos/{owner_namespace}/{package}/releases/tags/v{version}",
# # Combine with {"prerelease": true}
# template_staging_url="https://api.github.com/repos/{owner_namespace}/{package}/releases",
@@ -115,6 +119,7 @@ class DistributionPlatform(enum.Enum):
# )
MAVEN = DistributionPlatformValue(
name="Maven Central",
+ gh_slug="maven",
template_url="https://search.maven.org/solrsearch/select?q=g:{owner_namespace}+AND+a:{package}+AND+v:{version}&core=gav&rows=20&wt=json",
# Java ASF projects use staging URLs along the lines of
# https://repository.apache.org/content/repositories/orgapachePROJECT-NNNN/
@@ -123,17 +128,20 @@ class DistributionPlatform(enum.Enum):
)
NPM = DistributionPlatformValue(
name="npm",
+ gh_slug="npm",
# TODO: Need to parse dist-tags
template_url="https://registry.npmjs.org/{package}",
)
NPM_SCOPED = DistributionPlatformValue(
name="npm (scoped)",
+ gh_slug="npm",
# TODO: Need to parse dist-tags
template_url="https://registry.npmjs.org/@{owner_namespace}/{package}",
requires_owner_namespace=True,
)
PYPI = DistributionPlatformValue(
name="PyPI",
+ gh_slug="pypi",
template_url="https://pypi.org/pypi/{package}/{version}/json",
template_staging_url="https://test.pypi.org/pypi/{package}/{version}/json",
)
@@ -179,7 +187,7 @@ class TaskStatus(str, enum.Enum):
class TaskType(str, enum.Enum):
- GITHUB_ACTION_WORKFLOW = "github_action_workflow"
+ DISTRIBUTION_WORKFLOW = "distribution_workflow"
HASHING_CHECK = "hashing_check"
KEYS_IMPORT_FILE = "keys_import_file"
LICENSE_FILES = "license_files"
diff --git a/atr/post/distribution.py b/atr/post/distribution.py
index 15fa395..c5f3428 100644
--- a/atr/post/distribution.py
+++ b/atr/post/distribution.py
@@ -25,6 +25,9 @@ import atr.shared as shared
import atr.storage as storage
import atr.web as web
+_AUTOMATED_PLATFORMS = [shared.distribution.DistributionPlatform.MAVEN]
+_AUTOMATED_PLATFORMS_STAGE = [shared.distribution.DistributionPlatform.MAVEN]
+
@post.committer("/distribution/delete/<project>/<version>")
@post.form(shared.distribution.DeleteForm)
@@ -53,12 +56,84 @@ async def delete(
)
return await session.redirect(
get.distribution.list_get,
- project=project,
- version=version,
+ project_name=project,
+ version_name=version,
success="Distribution deleted",
)
+async def automate_form_process_page(
+ session: web.Committer,
+ form_data: shared.distribution.DistributeForm,
+ project: str,
+ version: str,
+ /,
+ staging: bool = False,
+) -> web.WerkzeugResponse:
+ allowed_platforms = _AUTOMATED_PLATFORMS_STAGE if staging else _AUTOMATED_PLATFORMS
+ if form_data.platform not in allowed_platforms:
+ platform_str = form_data.platform.value
+ return await session.redirect(
+ get.distribution.stage_automate if staging else get.distribution.automate,
+ project=project,
+ version=version,
+ error=f"Platform {platform_str} is not supported for automated distribution",
+ )
+ sql_platform = form_data.platform.to_sql() # type: ignore[attr-defined]
+ dd = distribution.Data(
+ platform=sql_platform,
+ owner_namespace=form_data.owner_namespace,
+ package=form_data.package,
+ version=form_data.version,
+ details=form_data.details,
+ )
+ release, committee = await shared.distribution.release_validated_and_committee(
+ project, version, staging=staging, release_policy=True
+ )
+ if release.release_policy is None or release.release_policy.github_repository_name == "":
+ return await session.redirect(
+ get.distribution.stage_automate if staging else get.distribution.automate,
+ project=project,
+ version=version,
+ error="Project does not have a release policy configured, or no GitHub repository is specified",
+ )
+ repo = release.release_policy.github_repository_name
+ workflow = f"distribute-{sql_platform.value.gh_slug}.yml"
+ async with storage.write_as_committee_member(committee_name=committee.name) as w:
+ try:
+ await w.distributions.automate(
+ release.name,
+ dd.platform,
+ dd.owner_namespace,
+ "apache",
+ repo,
+ workflow,
+ project,
+ version,
+ release.latest_revision_number,
+ dd.package,
+ dd.version,
+ staging,
+ )
+ except storage.AccessError as e:
+ # Instead of calling record_form_page_new, redirect with error message
+ return await session.redirect(
+ get.distribution.stage_automate if staging else get.distribution.automate,
+ project=project,
+ version=version,
+ error=str(e),
+ )
+
+ # Success - redirect to distribution list with success message
+ message = "Distribution queued successfully."
+ return await session.redirect(
+ get.distribution.list_get if staging else get.finish.selected,
+ project_name=project,
+ version_name=version,
+ success=message,
+ )
+
+
async def record_form_process_page(
session: web.Committer,
form_data: shared.distribution.DistributeForm,
@@ -84,14 +159,14 @@ async def record_form_process_page(
async with storage.write_as_committee_member(committee_name=committee.name) as w:
try:
_dist, added, _metadata = await w.distributions.record_from_data(
- release=release,
+ release=release.name,
staging=staging,
dd=dd,
)
except storage.AccessError as e:
# Instead of calling record_form_page_new, redirect with error message
return await session.redirect(
- get.distribution.stage if staging else get.distribution.record,
+ get.distribution.stage_record if staging else get.distribution.record,
project=project,
version=version,
error=str(e),
@@ -101,12 +176,28 @@ async def record_form_process_page(
message = "Distribution recorded successfully." if added else "Distribution was already recorded."
return await session.redirect(
get.distribution.list_get,
- project=project,
- version=version,
+ project_name=project,
+ version_name=version,
success=message,
)
+@post.committer("/distribution/automate/<project>/<version>")
+@post.form(shared.distribution.DistributeForm)
+async def automate_selected(
+ session: web.Committer, distribute_form: shared.distribution.DistributeForm, project: str, version: str
+) -> web.WerkzeugResponse:
+ return await automate_form_process_page(session, distribute_form, project, version, staging=False)
+
+
+@post.committer("/distribution/stage/automate/<project>/<version>")
+@post.form(shared.distribution.DistributeForm)
+async def stage_automate_selected(
+ session: web.Committer, distribute_form: shared.distribution.DistributeForm, project: str, version: str
+) -> web.WerkzeugResponse:
+ return await automate_form_process_page(session, distribute_form, project, version, staging=True)
+
+
@post.committer("/distribution/record/<project>/<version>")
@post.form(shared.distribution.DistributeForm)
async def record_selected(
@@ -115,9 +206,9 @@ async def record_selected(
return await record_form_process_page(session, distribute_form, project, version, staging=False)
-@post.committer("/distribution/stage/<project>/<version>")
+@post.committer("/distribution/stage/record/<project>/<version>")
@post.form(shared.distribution.DistributeForm)
-async def stage_selected(
+async def stage_record_selected(
session: web.Committer, distribute_form: shared.distribution.DistributeForm, project: str, version: str
) -> web.WerkzeugResponse:
return await record_form_process_page(session, distribute_form, project, version, staging=True)
diff --git a/atr/server.py b/atr/server.py
index e39ffff..3f13739 100644
--- a/atr/server.py
+++ b/atr/server.py
@@ -335,9 +335,10 @@ def _app_setup_security_headers(app: base.QuartApp) -> None:
# Both object-src 'none' and base-uri 'none' are required by ASVS v5 3.4.3 (L2)
# The frame-ancestors 'none' directive is required by ASVS v5 3.4.6 (L2)
# Bootstrap uses data: URLs extensively, so we need to include that in img-src
+ # The script hash allows window.location.reload() and nothing else
csp_directives = [
"default-src 'self'",
- "script-src 'self'",
+ "script-src 'self' 'sha256-4TpZ3Tx5SLybDXPQaSHGuP1RU4D+pzck+02JLVY61BY=' 'unsafe-hashes'",
"style-src 'self' 'unsafe-inline'",
"img-src 'self' https://apache.org https://incubator.apache.org https://www.apache.org data:",
"font-src 'self'",
diff --git a/atr/shared/distribution.py b/atr/shared/distribution.py
index 444d066..743c781 100644
--- a/atr/shared/distribution.py
+++ b/atr/shared/distribution.py
@@ -132,10 +132,7 @@ def html_tr_a(label: str, value: str | None) -> htm.Element:
async def release_validated(
- project: str,
- version: str,
- committee: bool = False,
- staging: bool | None = None,
+ project: str, version: str, committee: bool = False, staging: bool | None = None, release_policy: bool = False
) -> sql.Release:
match staging:
case True:
@@ -149,6 +146,7 @@ async def release_validated(
project_name=project,
version=version,
_committee=committee,
+ _release_policy=release_policy,
).demand(RuntimeError(f"Release {project} {version} not found"))
if release.phase not in phase:
raise RuntimeError(f"Release {project} {version} is not in {phase}")
@@ -158,12 +156,9 @@ async def release_validated(
async def release_validated_and_committee(
- project: str,
- version: str,
- *,
- staging: bool | None = None,
+ project: str, version: str, *, staging: bool | None = None, release_policy: bool = False
) -> tuple[sql.Release, sql.Committee]:
- release = await release_validated(project, version, committee=True, staging=staging)
+ release = await release_validated(project, version, committee=True, staging=staging, release_policy=release_policy)
committee = release.committee
if committee is None:
raise RuntimeError(f"Release {project} {version} has no committee")
diff --git a/atr/storage/writers/distributions.py b/atr/storage/writers/distributions.py
index 2db5d85..a9efd84 100644
--- a/atr/storage/writers/distributions.py
+++ b/atr/storage/writers/distributions.py
@@ -31,6 +31,8 @@ import atr.models.distribution as distribution
import atr.models.sql as sql
import atr.storage as storage
import atr.storage.outcome as outcome
+import atr.tasks.gha as gha
+import atr.util as util
class GeneralPublic:
@@ -95,6 +97,50 @@ class CommitteeMember(CommitteeParticipant):
self.__asf_uid = asf_uid
self.__committee_name = committee_name
+ async def automate(
+ self,
+ release_name: str,
+ platform: sql.DistributionPlatform,
+ owner_namespace: str | None,
+ owner: str,
+ repo: str,
+ workflow: str,
+ project_name: str,
+ version_name: str,
+ revision_number: str | None,
+ package: str,
+ version: str,
+ staging: bool,
+ ) -> sql.Task:
+ dist_task = sql.Task(
+ task_type=sql.TaskType.DISTRIBUTION_WORKFLOW,
+ task_args=gha.DistributionWorkflow(
+ name=release_name,
+ namespace=owner_namespace or "",
+ package=package,
+ version=version,
+ project_name=project_name,
+ version_name=version_name,
+ platform=platform.name,
+ owner=owner,
+ repo=repo,
+ ref="main", # TODO: Un-hardcode
+ workflow=workflow,
+ staging=staging,
+ arguments={},
+ ).model_dump(),
+ asf_uid=util.unwrap(self.__asf_uid),
+ added=datetime.datetime.now(datetime.UTC),
+ status=sql.TaskStatus.QUEUED,
+ project_name=project_name,
+ version_name=version_name,
+ revision_number=revision_number,
+ )
+ self.__data.add(dist_task)
+ await self.__data.commit()
+ await self.__data.refresh(dist_task)
+ return dist_task
+
async def record(
self,
release_name: str,
@@ -148,7 +194,7 @@ class CommitteeMember(CommitteeParticipant):
async def record_from_data(
self,
- release: sql.Release,
+ release: str,
staging: bool,
dd: distribution.Data,
) -> tuple[sql.Distribution, bool, distribution.Metadata]:
@@ -176,7 +222,7 @@ class CommitteeMember(CommitteeParticipant):
web_url=web_url,
)
dist, added = await self.record(
- release_name=release.name,
+ release_name=release,
platform=dd.platform,
owner_namespace=dd.owner_namespace,
package=dd.package,
diff --git a/atr/tasks/__init__.py b/atr/tasks/__init__.py
index dde92ed..9d89e31 100644
--- a/atr/tasks/__init__.py
+++ b/atr/tasks/__init__.py
@@ -189,7 +189,7 @@ def queued(
def resolve(task_type: sql.TaskType) -> Callable[..., Awaitable[results.Results | None]]: # noqa: C901
match task_type:
- case sql.TaskType.GITHUB_ACTION_WORKFLOW:
+ case sql.TaskType.DISTRIBUTION_WORKFLOW:
return gha.trigger_workflow
case sql.TaskType.HASHING_CHECK:
return hashing.check
diff --git a/atr/tasks/gha.py b/atr/tasks/gha.py
index f64b2cc..7b9123f 100644
--- a/atr/tasks/gha.py
+++ b/atr/tasks/gha.py
@@ -24,48 +24,79 @@ import aiohttp
import atr.config as config
import atr.log as log
+import atr.models.distribution as distribution
import atr.models.results as results
import atr.models.schema as schema
+import atr.models.sql as sql
+
+# import atr.shared as shared
+import atr.storage as storage
import atr.tasks.checks as checks
_BASE_URL: Final[str] = "https://api.github.com/repos"
_IN_PROGRESS_STATUSES: Final[list[str]] = ["in_progress", "queued", "requested", "waiting", "pending", "expected"]
_COMPLETED_STATUSES: Final[list[str]] = ["completed"]
_FAILED_STATUSES: Final[list[str]] = ["failure", "startup_failure"]
-_TIMEOUT_S = 5
+_TIMEOUT_S = 60
-class GithubActionsWorkflow(schema.Strict):
+class DistributionWorkflow(schema.Strict):
"""Arguments for the task to start a Github Actions workflow."""
owner: str = schema.description("Github owner of the repository")
repo: str = schema.description("Repository in which to start the workflow")
- workflow_id: str = schema.description("Workflow ID")
ref: str = schema.description("Git ref to trigger the workflow")
+ workflow: str = schema.description("Workflow to trigger")
+ namespace: str = schema.description("Namespace to distribute to")
+ package: str = schema.description("Package to distribute")
+ version: str = schema.description("Version to distribute")
+ staging: bool = schema.description("Whether this is a staging distribution")
+ project_name: str = schema.description("Project name in ATR")
+ version_name: str = schema.description("Version name in ATR")
+ platform: str = schema.description("Distribution platform")
arguments: dict[str, str] = schema.description("Workflow arguments")
name: str = schema.description("Name of the run")
-@checks.with_model(GithubActionsWorkflow)
-async def trigger_workflow(args: GithubActionsWorkflow) -> results.Results | None:
+@checks.with_model(DistributionWorkflow)
+async def trigger_workflow(args: DistributionWorkflow) -> results.Results | None:
unique_id = f"{args.name}-{uuid.uuid4()}"
- payload = {"ref": args.ref, "inputs": {"atr-id": unique_id, **args.arguments}}
+ # release, committee = await shared.distribution.release_validated_and_committee(
+ # args.project,
+ # args.version,
+ # staging=True, # TODO: Un-hardcode
+ # )
+ try:
+ sql_platform = sql.DistributionPlatform[args.platform]
+ except KeyError:
+ _fail(f"Invalid platform: {args.platform}")
+ payload = {
+ "ref": args.ref,
+ "inputs": {
+ "atr-id": unique_id,
+ "platform": args.platform,
+ "distribution-package": args.package,
+ "distribution-version": args.version,
+ "staging": "true" if args.staging else "false",
+ **args.arguments,
+ },
+ }
headers = {"Accept": "application/vnd.github+json", "Authorization": f"Bearer {config.get().GITHUB_TOKEN}"}
log.info(
- f"Triggering Github workflow {args.owner}/{args.repo}/{args.workflow_id} with args: {
+ f"Triggering Github workflow {args.owner}/{args.repo}/{args.workflow} with args: {
json.dumps(args.arguments, indent=2)
}"
)
async with aiohttp.ClientSession() as session:
try:
async with session.post(
- f"{_BASE_URL}/{args.owner}/{args.repo}/actions/workflows/{args.workflow_id}/dispatches",
+ f"{_BASE_URL}/{args.owner}/{args.repo}/actions/workflows/{args.workflow}/dispatches",
headers=headers,
json=payload,
) as response:
response.raise_for_status()
except aiohttp.ClientResponseError as e:
- _fail(f"Failed to trigger workflow run: {e.message} ({e.status})")
+ _fail(f"Failed to trigger GitHub workflow: {e.message} ({e.status})")
run, run_id = await _find_triggered_run(session, args, headers, unique_id)
@@ -73,13 +104,46 @@ async def trigger_workflow(args: GithubActionsWorkflow) -> results.Results | Non
run = await _wait_for_completion(session, args, headers, run_id, unique_id)
if run.get("status") in _FAILED_STATUSES:
- _fail(f"Github workflow {args.owner}/{args.repo}/{args.workflow_id} run {run_id} failed with error")
+ _fail(f"Github workflow {args.owner}/{args.repo}/{args.workflow} run {run_id} failed with error")
if run.get("status") in _COMPLETED_STATUSES:
- log.info(f"Workflow {args.owner}/{args.repo}/{args.workflow_id} run {run_id} completed successfully")
- return results.GithubActionsWorkflow(
- kind="github_actions_workflow", name=args.name, run_id=run_id, url=run.get("html_url", "")
+ log.info(f"Workflow {args.owner}/{args.repo}/{args.workflow} run {run_id} completed successfully")
+ await _record_distribution(
+ "committee.name",
+ "release",
+ sql_platform,
+ namespace=args.namespace,
+ package=args.package,
+ version=args.version,
+ staging=True,
+ )
+ return results.DistributionWorkflow(
+ kind="distribution_workflow", name=args.name, run_id=run_id, url=run.get("html_url", "")
)
- _fail(f"Timed out waiting for workflow {args.owner}/{args.repo}/{args.workflow_id}")
+ _fail(f"Timed out waiting for GitHub workflow {args.owner}/{args.repo}/{args.workflow}")
+
+
+async def _record_distribution(
+ committee_name: str,
+ release: str,
+ platform: sql.DistributionPlatform,
+ namespace: str,
+ package: str,
+ version: str,
+ staging: bool,
+):
+ log.info("Creating distribution record")
+ dd = distribution.Data(
+ platform=platform,
+ owner_namespace=namespace,
+ package=package,
+ version=version,
+ details=False,
+ )
+ async with storage.write_as_committee_member(committee_name=committee_name) as w:
+ try:
+ _dist, _added, _metadata = await w.distributions.record_from_data(release=release, staging=staging, dd=dd)
+ except storage.AccessError as e:
+ _fail(f"Failed to record distribution: {e}")
def _fail(message: str) -> NoReturn:
@@ -89,7 +153,7 @@ def _fail(message: str) -> NoReturn:
async def _find_triggered_run(
session: aiohttp.ClientSession,
- args: GithubActionsWorkflow,
+ args: DistributionWorkflow,
headers: dict[str, str],
unique_id: str,
) -> tuple[dict[str, Any], int]:
@@ -140,7 +204,7 @@ async def _request_and_retry(
async def _wait_for_completion(
session: aiohttp.ClientSession,
- args: GithubActionsWorkflow,
+ args: DistributionWorkflow,
headers: dict[str, str],
run_id: int,
unique_id: str,
diff --git a/atr/templates/check-selected.html b/atr/templates/check-selected.html
index 0bb1534..77b793d 100644
--- a/atr/templates/check-selected.html
+++ b/atr/templates/check-selected.html
@@ -125,9 +125,19 @@
</div>
</div>
{% if phase == "release_candidate_draft" %}
+ <h3 id="distribution" class="mt-4">Distribution</h3>
+ <p>
+ While this release is in draft, you can create a staging distribution. Use the buttons below to either create one automatically (where supported) or record a manual distribution performed outside of ATR.
+ </p>
+ <div class="alert alert-warning mb-4">
+ <p class="fw-semibold mb-1">NOTE:</p>
+ <p>At present, automated distributions are being developed. Please use the manual record button if you need to record a distribution.</p>
+ </div>
<p>
<a class="btn btn-primary"
- href="{{ as_url(get.distribution.stage, project=release.project.name, version=release.version) }}">Record a distribution</a>
+ href="{{ as_url(get.distribution.stage_automate, project=release.project.name, version=release.version) }}">Distribute</a>
+ <a class="btn btn-secondary"
+ href="{{ as_url(get.distribution.stage_record, project=release.project.name, version=release.version) }}">Record a manual distribution</a>
</p>
<h2 id="more-actions">More actions</h2>
<h3 id="ignored-checks" class="mt-4">Ignored checks</h3>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org