You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tooling.apache.org by sb...@apache.org on 2025/06/10 14:40:54 UTC
(tooling-trusted-release) branch main updated: Add a release consistency check page for admins
This is an automated email from the ASF dual-hosted git repository.
sbp pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tooling-trusted-release.git
The following commit(s) were added to refs/heads/main by this push:
new a474919 Add a release consistency check page for admins
a474919 is described below
commit a4749193a9e9da8628ca13debf89caaed3546efe
Author: Sean B. Palmer <se...@miscoranda.com>
AuthorDate: Tue Jun 10 15:38:48 2025 +0100
Add a release consistency check page for admins
---
atr/blueprints/admin/admin.py | 82 ++++++++++++++++++++++++++++++++++++
atr/util.py | 97 ++++++++++++++++++-------------------------
2 files changed, 122 insertions(+), 57 deletions(-)
diff --git a/atr/blueprints/admin/admin.py b/atr/blueprints/admin/admin.py
index 54f3a5f..7180b9e 100644
--- a/atr/blueprints/admin/admin.py
+++ b/atr/blueprints/admin/admin.py
@@ -97,6 +97,55 @@ class LdapLookupForm(util.QuartFormTyped):
submit = wtforms.SubmitField("Lookup")
+@admin.BLUEPRINT.route("/consistency")
+async def admin_consistency() -> quart.Response:
+ """Check for consistency between the database and the filesystem."""
+ # Get all releases from the database
+ async with db.session() as data:
+ releases = await data.release().all()
+ database_dirs = []
+ for release in releases:
+ path = util.release_directory_version(release)
+ if await aiofiles.os.path.exists(path):
+ database_dirs.append(str(path))
+ if len(set(database_dirs)) != len(database_dirs):
+ raise base.ASFQuartException("Duplicate release directories in database", errorcode=500)
+
+ # Get all releases from the filesystem
+ filesystem_dirs = await _get_filesystem_dirs()
+
+ # Pair them up where possible
+ paired_dirs = []
+ for database_dir in database_dirs[:]:
+ for filesystem_dir in filesystem_dirs[:]:
+ if database_dir == filesystem_dir:
+ paired_dirs.append(database_dir)
+ database_dirs.remove(database_dir)
+ filesystem_dirs.remove(filesystem_dir)
+ break
+ return quart.Response(
+ f"""\
+=== BROKEN ===
+
+DATABASE ONLY:
+
+{"\n".join(sorted(database_dirs or ["-"]))}
+
+FILESYSTEM ONLY:
+
+{"\n".join(sorted(filesystem_dirs or ["-"]))}
+
+
+== Okay ==
+
+Paired correctly:
+
+{"\n".join(sorted(paired_dirs or ["-"]))}
+""",
+ mimetype="text/plain",
+ )
+
+
@admin.BLUEPRINT.route("/data")
@admin.BLUEPRINT.route("/data/<model>")
async def admin_data(model: str = "Committee") -> str:
@@ -542,6 +591,39 @@ async def _delete_release_data(release_name: str) -> None:
)
+async def _get_filesystem_dirs() -> list[str]:
+ filesystem_dirs = []
+ await _get_filesystem_dirs_finished(filesystem_dirs)
+ await _get_filesystem_dirs_unfinished(filesystem_dirs)
+ return filesystem_dirs
+
+
+async def _get_filesystem_dirs_finished(filesystem_dirs: list[str]) -> None:
+ finished_dir = util.get_finished_dir()
+ finished_dir_contents = await aiofiles.os.listdir(finished_dir)
+ for project_dir in finished_dir_contents:
+ project_dir_path = os.path.join(finished_dir, project_dir)
+ if await aiofiles.os.path.isdir(project_dir_path):
+ for version_dir in await aiofiles.os.listdir(project_dir_path):
+ if await aiofiles.os.path.isdir(os.path.join(project_dir_path, version_dir)):
+ version_dir_path = os.path.join(project_dir_path, version_dir)
+ if await aiofiles.os.path.isdir(version_dir_path):
+ filesystem_dirs.append(version_dir_path)
+
+
+async def _get_filesystem_dirs_unfinished(filesystem_dirs: list[str]) -> None:
+ unfinished_dir = util.get_unfinished_dir()
+ unfinished_dir_contents = await aiofiles.os.listdir(unfinished_dir)
+ for project_dir in unfinished_dir_contents:
+ project_dir_path = os.path.join(unfinished_dir, project_dir)
+ if await aiofiles.os.path.isdir(project_dir_path):
+ for version_dir in await aiofiles.os.listdir(project_dir_path):
+ if await aiofiles.os.path.isdir(os.path.join(project_dir_path, version_dir)):
+ version_dir_path = os.path.join(project_dir_path, version_dir)
+ if await aiofiles.os.path.isdir(version_dir_path):
+ filesystem_dirs.append(version_dir_path)
+
+
async def _process_undiscovered(data: db.Session) -> tuple[int, int]:
added_count = 0
updated_count = 0
diff --git a/atr/util.py b/atr/util.py
index 3492d02..c26e22b 100644
--- a/atr/util.py
+++ b/atr/util.py
@@ -388,64 +388,14 @@ def is_user_viewing_as_admin(uid: str | None) -> bool:
async def number_of_release_files(release: models.Release) -> int:
"""Return the number of files in a release."""
- path_project = release.project.name
- path_version = release.version
- path_revision = release.latest_revision_number
- if path_revision is None:
+ if (path := release_directory_revision(release)) is None:
return 0
- match release.phase:
- case models.ReleasePhase.RELEASE_CANDIDATE_DRAFT:
- path = get_unfinished_dir() / path_project / path_version / path_revision
- case models.ReleasePhase.RELEASE_CANDIDATE:
- path = get_unfinished_dir() / path_project / path_version / path_revision
- case models.ReleasePhase.RELEASE_PREVIEW:
- path = get_unfinished_dir() / path_project / path_version / path_revision
- case models.ReleasePhase.RELEASE:
- path = get_finished_dir() / path_project / path_version
- case _:
- raise ValueError(f"Unknown release phase: {release.phase}")
count = 0
async for _ in paths_recursive(path):
count += 1
return count
-async def number_of_release_files_debugging(release: models.Release) -> tuple[int, dict[str, str]]:
- """Return the number of files in a release."""
- debugging = {}
- path_project = release.project.name
- debugging["path_project"] = path_project
- path_version = release.version
- debugging["path_version"] = path_version
- path_revision = release.latest_revision_number
- debugging["path_revision"] = str(path_revision)
- if path_revision is None:
- debugging["reason"] = "path_revision is None"
- return 0, debugging
- match release.phase:
- case models.ReleasePhase.RELEASE_CANDIDATE_DRAFT:
- debugging["matched"] = "RELEASE_CANDIDATE_DRAFT"
- path = get_unfinished_dir() / path_project / path_version / path_revision
- case models.ReleasePhase.RELEASE_CANDIDATE:
- debugging["matched"] = "RELEASE_CANDIDATE"
- path = get_unfinished_dir() / path_project / path_version / path_revision
- case models.ReleasePhase.RELEASE_PREVIEW:
- debugging["matched"] = "RELEASE_PREVIEW"
- path = get_unfinished_dir() / path_project / path_version / path_revision
- case models.ReleasePhase.RELEASE:
- debugging["matched"] = "RELEASE"
- path = get_finished_dir() / path_project / path_version
- case _:
- raise ValueError(f"Unknown release phase: {release.phase}")
- debugging["path"] = str(path)
- count = 0
- debugging["stat"] = str(await aiofiles.os.stat(path))
- async for _ in paths_recursive(path):
- count += 1
- debugging["count"] = str(count)
- return count, debugging
-
-
def parse_key_blocks(keys_text: str) -> list[str]:
"""Extract GPG key blocks from a KEYS file."""
key_blocks = []
@@ -565,10 +515,8 @@ def release_directory(release: models.Release) -> pathlib.Path:
def release_directory_base(release: models.Release) -> pathlib.Path:
"""Determine the filesystem directory for a given release based on its phase."""
phase = release.phase
- try:
- project_name, version_name = release.name.rsplit("-", 1)
- except ValueError:
- raise base.ASFQuartException(f"Invalid release name format '{release.name}'", 500)
+ project_name = release.project.name
+ version_name = release.version
base_dir: pathlib.Path | None = None
match phase:
@@ -580,11 +528,46 @@ def release_directory_base(release: models.Release) -> pathlib.Path:
base_dir = get_unfinished_dir()
case models.ReleasePhase.RELEASE:
base_dir = get_finished_dir()
- # NOTE: Do NOT add "case _" here
-
+ # Do not add "case _" here
return base_dir / project_name / version_name
+def release_directory_revision(release: models.Release) -> pathlib.Path | None:
+ """Return the path to the directory containing the active files for a given release phase."""
+ path_project = release.project.name
+ path_version = release.version
+ match release.phase:
+ case (
+ models.ReleasePhase.RELEASE_CANDIDATE_DRAFT
+ | models.ReleasePhase.RELEASE_CANDIDATE
+ | models.ReleasePhase.RELEASE_PREVIEW
+ ):
+ if (path_revision := release.latest_revision_number) is None:
+ return None
+ path = get_unfinished_dir() / path_project / path_version / path_revision
+ case models.ReleasePhase.RELEASE:
+ path = get_finished_dir() / path_project / path_version
+ # Do not add "case _" here
+ return path
+
+
+def release_directory_version(release: models.Release) -> pathlib.Path:
+ """Return the path to the directory containing the active files for a given release phase."""
+ path_project = release.project.name
+ path_version = release.version
+ match release.phase:
+ case (
+ models.ReleasePhase.RELEASE_CANDIDATE_DRAFT
+ | models.ReleasePhase.RELEASE_CANDIDATE
+ | models.ReleasePhase.RELEASE_PREVIEW
+ ):
+ path = get_unfinished_dir() / path_project / path_version
+ case models.ReleasePhase.RELEASE:
+ path = get_finished_dir() / path_project / path_version
+ # Do not add "case _" here
+ return path
+
+
def unwrap(value: T | None, error_message: str = "unexpected None when unwrapping value") -> T:
"""
Will unwrap the given value or raise a ValueError if it is None
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org