You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tooling.apache.org by sb...@apache.org on 2026/08/13 14:06:56 UTC
(tooling-trusted-releases) 01/01: Queue the finalisation task on announcement locally instead of moving files
This is an automated email from the ASF dual-hosted git repository.
sbp pushed a commit to branch sbp
in repository https://gitbox.apache.org/repos/asf/tooling-trusted-releases.git
commit 0ad53294855fab89e219ccea9d53638505b619f8
Author: Sean B. Palmer <se...@miscoranda.com>
AuthorDate: Thu Aug 13 15:06:06 2026 +0100
Queue the finalisation task on announcement locally instead of moving files
---
atr/config.py | 5 ++
atr/log.py | 12 +++
atr/storage/__init__.py | 4 +-
atr/storage/writers/announce.py | 117 +++++++++++++++++++++--------
tests/docker-compose.yml | 1 +
tests/unit/test_announce_deletion_order.py | 52 ++++++++++++-
6 files changed, 151 insertions(+), 40 deletions(-)
diff --git a/atr/config.py b/atr/config.py
index cfe6c1a5..8f0c8159 100644
--- a/atr/config.py
+++ b/atr/config.py
@@ -130,6 +130,11 @@ class AppConfig:
# The dist pubsub watcher catalogues non-ATR releases it sees. Off by default, so
# it runs in report mode (logs what it would create); set true to write to the db.
DIST_CATALOG_WRITE = _ENVIRONMENT_CONFIG("DIST_CATALOG_WRITE", default=False, cast=bool)
+ RELEASE_FINALISE = _ENVIRONMENT_CONFIG(
+ "RELEASE_FINALISE",
+ cast=bool,
+ default=APP_HOST.split(":", 1)[0] in ("127.0.0.1", "atr", "localhost.apache.org"),
+ )
SVN_TOKEN = _config_secrets("SVN_TOKEN", STATE_DIR, default=None, cast=str)
SVN_PUBLISH_URL = _config_secrets("SVN_PUBLISH_URL", STATE_DIR, default=None, cast=str)
SVN_DIST_PUBLIC_URL = _ENVIRONMENT_CONFIG("SVN_DIST_PUBLIC_URL", default="https://dist.apache.org/repos/dist/atr")
diff --git a/atr/log.py b/atr/log.py
index 57c4413d..b5414111 100644
--- a/atr/log.py
+++ b/atr/log.py
@@ -60,6 +60,18 @@ def add_context(**kwargs):
structlog.contextvars.bind_contextvars(**kwargs)
+def audit_datetime() -> str:
+ now = datetime.datetime.now(datetime.UTC).isoformat(timespec="milliseconds")
+ return now.replace("+00:00", "Z")
+
+
+def audit_flush() -> None:
+ for handler in logging.getLogger("atr.storage.audit").handlers:
+ handler_queue = getattr(handler, "queue", None)
+ if handler_queue is not None:
+ handler_queue.join()
+
+
def auth_event(event: str, asfuid: str | None = None, **kwargs):
request_user_id = get_context("user_id")
request_asf_id = get_context("asfuid")
diff --git a/atr/storage/__init__.py b/atr/storage/__init__.py
index c0e52f23..0305a3c9 100644
--- a/atr/storage/__init__.py
+++ b/atr/storage/__init__.py
@@ -18,7 +18,6 @@
from __future__ import annotations
import contextlib
-import datetime
import json
import logging
from typing import TYPE_CHECKING, Final, Protocol
@@ -575,8 +574,7 @@ class Write:
# Do not rename this interface
# It is named to reserve the atr.storage.audit logger name
def audit(**kwargs: basic.JSON) -> None:
- now = datetime.datetime.now(datetime.UTC).isoformat(timespec="milliseconds")
- now = now.replace("+00:00", "Z")
+ now = log.audit_datetime()
action = log.caller_name(depth=2)
request_user_id = log.get_context("user_id")
admin_user_id = log.get_context("admin_id")
diff --git a/atr/storage/writers/announce.py b/atr/storage/writers/announce.py
index ef65575c..24222bde 100644
--- a/atr/storage/writers/announce.py
+++ b/atr/storage/writers/announce.py
@@ -30,6 +30,7 @@ import aioshutil
import sqlmodel
import atr.analysis as analysis
+import atr.auditlog as auditlog
import atr.catalog_site as catalog_site
import atr.config as config
import atr.construct as construct
@@ -208,15 +209,19 @@ class ReleaseManager(CommitteeParticipant):
subject, _ = await construct.announce_release_subject_and_body(subject_template, "", options)
# Prepare paths for file operations
+ finalise = config.get().RELEASE_FINALISE
unfinished_revisions_path = paths.release_directory_base(release)
unfinished_path = unfinished_revisions_path / release.unwrap_revision_number
unfinished_dir = str(unfinished_path)
release_date = datetime.datetime.now(datetime.UTC)
- predicted_finished_release = self.__predicted_finished_release(release, release_date)
- finished_path = paths.release_directory(predicted_finished_release)
- finished_dir = str(finished_path)
- if await aiofiles.os.path.exists(finished_dir):
- raise storage.AccessError("Release already exists", status=409)
+ artifact_files_path = unfinished_path
+ if not finalise:
+ predicted_finished_release = self.__predicted_finished_release(release, release_date)
+ finished_path = paths.release_directory(predicted_finished_release)
+ finished_dir = str(finished_path)
+ artifact_files_path = finished_path
+ if await aiofiles.os.path.exists(finished_dir):
+ raise storage.AccessError("Release already exists", status=409)
# TODO: This is not reliable because of race conditions
# But it adds a layer of protection in most cases
completed_publish = await interaction.release_completed_svn_publish_task_for_revision(
@@ -259,29 +264,71 @@ class ReleaseManager(CommitteeParticipant):
status=503,
)
- # Ensure that the permissions of every directory are 755
- await asyncio.to_thread(util.chmod_directories, unfinished_path)
-
- try:
- # Move the release files from somewhere in unfinished to somewhere in finished
- # The whole finished hierarchy is write once for each directory, and then read only
- # TODO: Set permissions to help enforce this, or find alternative methods
- await aioshutil.move(unfinished_dir, finished_dir)
+ audit_stamp = log.audit_datetime()
+ finalise_args: args.ReleaseFinalise | None = None
+ if finalise:
+ if published_revision is None:
+ raise storage.AccessError(
+ "The completed SVN publish has no recorded revision - Invalid state", status=500
+ )
+ finalise_args = args.ReleaseFinalise(
+ asf_uid=self.__asf_uid,
+ project_key=project_key,
+ version_key=version_key,
+ revision_number=preview_revision_number,
+ svn_revision=published_revision,
+ download_path_suffix=effective_download_path_suffix,
+ audit_until=audit_stamp,
+ )
self.__write_as.append_to_audit_log(
+ action=auditlog.RELEASE_ANNOUNCE_ACTION,
+ datetime=audit_stamp,
asf_uid=self.__asf_uid,
project_key=str(project_key),
version_key=str(version_key),
revision_number=str(preview_revision_number),
- source_directory=unfinished_dir,
- target_directory=finished_dir,
email_to=email_to,
email_cc=basic.as_json(email_cc or []),
email_bcc=basic.as_json(email_bcc or []),
)
- except Exception as e:
- raise storage.AccessError(f"Error moving files: {e!s}", status=500)
+ await asyncio.to_thread(log.audit_flush)
+ else:
+ # Ensure that the permissions of every directory are 755
+ await asyncio.to_thread(util.chmod_directories, unfinished_path)
+ try:
+ # Move the release files from somewhere in unfinished to somewhere in finished
+ # The whole finished hierarchy is write once for each directory, and then read only
+ # TODO: Set permissions to help enforce this, or find alternative methods
+ await aioshutil.move(unfinished_dir, finished_dir)
+ self.__write_as.append_to_audit_log(
+ action=auditlog.RELEASE_ANNOUNCE_ACTION,
+ datetime=audit_stamp,
+ asf_uid=self.__asf_uid,
+ project_key=str(project_key),
+ version_key=str(version_key),
+ revision_number=str(preview_revision_number),
+ source_directory=unfinished_dir,
+ target_directory=finished_dir,
+ email_to=email_to,
+ email_cc=basic.as_json(email_cc or []),
+ email_bcc=basic.as_json(email_bcc or []),
+ )
+ except Exception as e:
+ raise storage.AccessError(f"Error moving files: {e!s}", status=500)
try:
+ if finalise_args is not None:
+ self.__data.add(
+ sql.Task(
+ status=sql.TaskStatus.QUEUED,
+ task_type=sql.TaskType.RELEASE_FINALISE,
+ task_args=finalise_args.model_dump(),
+ asf_uid=self.__asf_uid,
+ project_key=str(project_key),
+ version_key=str(version_key),
+ revision_number=str(preview_revision_number),
+ )
+ )
task = sql.Task(
status=sql.TaskStatus.QUEUED,
task_type=sql.TaskType.MESSAGE_SEND,
@@ -323,7 +370,7 @@ class ReleaseManager(CommitteeParticipant):
await self.__write_artifact_rows(
release,
committee,
- finished_path,
+ artifact_files_path,
preview_revision_number,
published_revision,
effective_download_path_suffix,
@@ -345,23 +392,27 @@ class ReleaseManager(CommitteeParticipant):
except storage.AccessError:
raise
except Exception as e:
+ if finalise:
+ raise storage.AccessError(f"Error queuing announcement: {e!s}", status=500)
raise storage.AccessError(
f"Files moved successfully, but error queuing announcement: {e!s}. Manual cleanup needed.",
status=500,
)
- try:
- if unfinished_revisions_path:
- # This removes all of the prior revisions
- # Each prior revision directory is immutable
- await util.delete_immutable_directory(
- unfinished_revisions_path,
- reason=f"user {self.__asf_uid} is releasing {project_key} {version_key} {preview_revision_number}",
+ if not finalise:
+ try:
+ if unfinished_revisions_path:
+ # This removes all of the prior revisions
+ # Each prior revision directory is immutable
+ await util.delete_immutable_directory(
+ unfinished_revisions_path,
+ reason=f"user {self.__asf_uid} is releasing"
+ f" {project_key} {version_key} {preview_revision_number}",
+ )
+ except Exception as e:
+ raise storage.AccessError(
+ f"Release announced, but error deleting prior revisions: {e!s}. Manual cleanup needed.",
+ status=500,
)
- except Exception as e:
- raise storage.AccessError(
- f"Release announced, but error deleting prior revisions: {e!s}. Manual cleanup needed.",
- status=500,
- )
await self.__archive_prior_release(release, auto_archive_prior)
async def __archive_prior_release(self, release_row: sql.Release, auto_archive_prior: bool) -> None:
@@ -640,7 +691,7 @@ class ReleaseManager(CommitteeParticipant):
self,
release: sql.Release,
committee: sql.Committee,
- finished_path: safe.StatePath,
+ files_path: safe.StatePath,
revision_number: safe.RevisionNumber,
svn_revision: int | None,
download_path_suffix: safe.RelPath | None,
@@ -650,7 +701,7 @@ class ReleaseManager(CommitteeParticipant):
# The preview revision created when a vote passes has no checks of its own, so the
# signature check sits on the parent draft it was promoted from
parent_revision_number = await self.__parent_revision_number(release.key, revision_number)
- rel_paths = {str(p) async for p in util.paths_recursive(finished_path)}
+ rel_paths = {str(p) async for p in util.paths_recursive(files_path)}
classifications = await self.__data.release_file_classifications_at(release.key, revision_seq)
# The directory the files publish to under the dist root, the same for every artifact here
dist_dir = str(paths.committee_dist_relpath(committee, download_path_suffix))
@@ -676,7 +727,7 @@ class ReleaseManager(CommitteeParticipant):
else None
)
signature_sha3 = (
- await hashes.file_sha3(str(finished_path / signature_path)) if (signature_path is not None) else None
+ await hashes.file_sha3(str(files_path / signature_path)) if (signature_path is not None) else None
)
self.__data.add(
sql.Artifact(
diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml
index 0219560d..c69ed141 100644
--- a/tests/docker-compose.yml
+++ b/tests/docker-compose.yml
@@ -28,6 +28,7 @@ services:
- TESTS=1
- APP_HOST=atr-dev:8080
- BIND=0.0.0.0:8080
+ - RELEASE_FINALISE=true
- SSH_HOST=0.0.0.0
networks:
- test-network
diff --git a/tests/unit/test_announce_deletion_order.py b/tests/unit/test_announce_deletion_order.py
index 40b12e35..93a34040 100644
--- a/tests/unit/test_announce_deletion_order.py
+++ b/tests/unit/test_announce_deletion_order.py
@@ -22,16 +22,25 @@ from typing import Any
import pytest
+import atr.models.args as args
import atr.models.safe as safe
+import atr.models.sql as sql
import atr.storage as storage
import atr.storage.writers.announce as announce
+def added_tasks(release_manager: announce.ReleaseManager, task_type: sql.TaskType) -> list[sql.Task]:
+ data = getattr(release_manager, "_ReleaseManager__data")
+ added = [call.args[0] for call in data.add.call_args_list]
+ return [task for task in added if getattr(task, "task_type", None) == task_type]
+
+
def announcing_writer(
- monkeypatch: pytest.MonkeyPatch, tmp_path: pathlib.Path, calls: list[str]
+ monkeypatch: pytest.MonkeyPatch, tmp_path: pathlib.Path, calls: list[str], finalise: bool = False
) -> announce.ReleaseManager:
unfinished = tmp_path / "unfinished" / "example" / "2.0.0"
(unfinished / "00003").mkdir(parents=True)
+ monkeypatch.setattr(announce.config.get(), "RELEASE_FINALISE", finalise)
monkeypatch.setattr(announce.aioshutil, "move", mock.AsyncMock(side_effect=lambda *a, **k: calls.append("move")))
monkeypatch.setattr(announce.config, "is_dev_environment", lambda: True)
monkeypatch.setattr(announce.config, "svn_publish_kind", lambda: announce.config.SvnPublishKind.ASF_DISTRIBUTION)
@@ -49,9 +58,10 @@ def announcing_writer(
monkeypatch.setattr(
announce.construct, "release_notification", lambda *a, **k: SimpleNamespace(as_task_args=lambda: {})
)
+ monkeypatch.setattr(announce.log, "audit_flush", lambda: calls.append("flush"))
monkeypatch.setattr(announce.paths, "release_directory", lambda _release: tmp_path / "finished" / "example")
monkeypatch.setattr(announce.paths, "release_directory_base", lambda _release: unfinished)
- monkeypatch.setattr(announce.util, "chmod_directories", lambda *a, **k: None)
+ monkeypatch.setattr(announce.util, "chmod_directories", lambda *a, **k: calls.append("chmod"))
monkeypatch.setattr(
announce.util,
"delete_immutable_directory",
@@ -65,6 +75,11 @@ def announcing_writer(
data.release.return_value.demand = mock.AsyncMock(return_value=release_row())
# A released version queues a catalog-site regeneration, which first looks for an existing queued one.
data.task.return_value.get = mock.AsyncMock(return_value=None)
+ data.add = mock.MagicMock(
+ side_effect=lambda obj: (
+ calls.append("finalise") if getattr(obj, "task_type", None) == sql.TaskType.RELEASE_FINALISE else None
+ )
+ )
data.commit = mock.AsyncMock(side_effect=lambda: calls.append("commit"))
release_manager = object.__new__(announce.ReleaseManager)
release_manager._ReleaseManager__data = data
@@ -139,7 +154,33 @@ async def test_commit_failure_skips_prior_revision_deletion(
with pytest.raises(storage.AccessError, match="Files moved successfully"):
await release_manager.release(**release_arguments())
- assert calls == ["move"]
+ assert calls == ["chmod", "move"]
+
+
+@pytest.mark.asyncio
+async def test_finalise_queues_task_and_keeps_files(monkeypatch: pytest.MonkeyPatch, tmp_path: pathlib.Path) -> None:
+ calls: list[str] = []
+ release_manager = announcing_writer(monkeypatch, tmp_path, calls, finalise=True)
+ monkeypatch.setattr(
+ announce.interaction,
+ "release_completed_svn_publish_task_for_revision",
+ mock.AsyncMock(return_value=SimpleNamespace(task_args={}, result={"svn_revision": 85})),
+ )
+
+ await release_manager.release(**release_arguments())
+
+ assert calls == ["flush", "finalise", "commit"]
+ tasks = added_tasks(release_manager, sql.TaskType.RELEASE_FINALISE)
+ assert len(tasks) == 1
+ task_args = tasks[0].task_args
+ parsed = args.ReleaseFinalise.model_validate(task_args)
+ assert parsed.svn_revision == 85
+ audit_call = release_manager._ReleaseManager__write_as.append_to_audit_log.call_args
+ assert audit_call.kwargs["action"] == "release_announce"
+ assert audit_call.kwargs["datetime"] == parsed.audit_until
+ rows_path = release_manager._ReleaseManager__write_artifact_rows.call_args.args[2]
+ assert pathlib.Path(str(rows_path)) == tmp_path / "unfinished" / "example" / "2.0.0" / "00003"
+ assert (tmp_path / "unfinished" / "example" / "2.0.0" / "00003").is_dir()
@pytest.mark.asyncio
@@ -149,4 +190,7 @@ async def test_prior_revisions_deleted_after_commit(monkeypatch: pytest.MonkeyPa
await release_manager.release(**release_arguments())
- assert calls == ["move", "commit", "delete"]
+ assert calls == ["chmod", "move", "commit", "delete"]
+ assert added_tasks(release_manager, sql.TaskType.RELEASE_FINALISE) == []
+ audit_call = release_manager._ReleaseManager__write_as.append_to_audit_log.call_args
+ assert audit_call.kwargs["action"] == "release_announce"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org