You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tooling.apache.org by ar...@apache.org on 2026/05/21 13:58:18 UTC

(tooling-trusted-releases) branch arm updated: #470 - add validation rules for new fields, update form validation for add project to use field validators

This is an automated email from the ASF dual-hosted git repository.

alitheg pushed a commit to branch arm
in repository https://gitbox.apache.org/repos/asf/tooling-trusted-releases.git


The following commit(s) were added to refs/heads/arm by this push:
     new 8372b3fe #470 - add validation rules for new fields, update form validation for add project to use field validators
8372b3fe is described below

commit 8372b3fe415e22a3562d76b853cdb73b8465a928
Author: Alastair McFarlane <ar...@apache.org>
AuthorDate: Thu May 21 14:58:00 2026 +0100

    #470 - add validation rules for new fields, update form validation for add project to use field validators
---
 atr/docs/input-validation.md     |  43 ++++++++++++++-
 atr/models/safe.py               |   4 ++
 atr/post/projects.py             |   6 +-
 atr/shared/projects.py           |  68 +++++++++++++++--------
 atr/storage/writers/project.py   |  22 ++++----
 atr/validate.py                  | 116 ++++++++++++++++++++++++++++++++++++++-
 tests/unit/test_policy_writer.py |  13 +++--
 7 files changed, 228 insertions(+), 44 deletions(-)

diff --git a/atr/docs/input-validation.md b/atr/docs/input-validation.md
index db4a5ed3..e169bbb1 100644
--- a/atr/docs/input-validation.md
+++ b/atr/docs/input-validation.md
@@ -166,21 +166,62 @@ Beyond input validation, ATR performs data integrity validation on database reco
 
 ### Committee validation
 
+Committee records are not created through a form. They are synced from LDAP and Whimsy by [`_update_committees`](/ref/atr/datasources/apache.py:_update_committees), so their fields have no Pydantic input layer, and the data integrity validators are the main place each field is confirmed.
+
 The [`committee`](/ref/atr/validate.py:committee) function checks:
 
+* `key` must be a valid committee key (the [`safe.CommitteeKey`](/ref/atr/models/safe.py:CommitteeKey) character set)
+* `name` must be set, trimmed, and not prefixed with "Apache "
+* `committee_members`, `committers`, and `release_managers` entries must each look like an ASF UID
 * `child_committees` must be empty (not used)
-* `full_name` must be set, trimmed, and not prefixed with "Apache "
+
+Per-field coverage:
+
+| Field | Input layer | Data integrity |
+| --- | --- | --- |
+| `key` | none (LDAP project name) | valid committee key |
+| `name` | none (Whimsy) | set, trimmed, no "Apache " prefix |
+| `is_podling` | set during sync | type-enforced (bool) |
+| `parent_committee_key` | none | foreign key |
+| `committee_members` | none (LDAP) | ASF UID format |
+| `committers` | none (LDAP) | ASF UID format |
+| `release_managers` | none | ASF UID format |
+| `child_committees` | none | must be empty |
 
 ### Project validation
 
+Projects do have an input layer. [`AddProjectForm`](/ref/atr/shared/projects.py:AddProjectForm) validates the key and display name at creation, and [`EditVersionSchemeForm`](/ref/atr/shared/projects.py:EditVersionSchemeForm) rejects a `version_pattern` or `cycle_match` that is not a valid regex (and a `cycle_match` with no capture group). The data integrity validators mirror these and cover fields that have no form.
+
 The [`project`](/ref/atr/validate.py:project) function checks:
 
+* `key` must be a valid project key (the [`safe.ProjectKey`](/ref/atr/models/safe.py:ProjectKey) character set)
 * `category` must use comma-separated labels without colons
 * `committee_key` must be set (project must be linked to a committee)
 * `created` timestamp must be in the past
+* `created_by`, if set, must look like an ASF UID
+* `cycle_match`, if set, must be a regex with at least one capture group
 * `full_name` must be set and start with "Apache "
 * `programming_languages` must use comma-separated labels without colons
 * `release_policy_id` must be None (not used)
+* `version_pattern`, if set, must be a compilable regex
+
+Per-field coverage:
+
+| Field | Input layer | Data integrity |
+| --- | --- | --- |
+| `key` | `safe.ProjectKey` (charset, lowercase) + `AddProjectForm` (committee-prefixed) | valid project key |
+| `name` | `AddProjectForm` (Apache prefix, case rules) | set, "Apache " prefix |
+| `status` | none (enum) | type-enforced (enum) |
+| `description` | `EditMetadataForm` (free text) | none |
+| `category` | `AddCategoryForm` (free text) | comma-separated, no colons |
+| `programming_languages` | `AddLanguageForm` (free text) | comma-separated, no colons |
+| `version_method` | `EditVersionSchemeForm` (enum) | type-enforced (enum) |
+| `version_pattern` | `EditVersionSchemeForm` (regex validity) | compilable regex |
+| `cycle_match` | `EditVersionSchemeForm` (regex and capture group) | regex with capture group |
+| `branch_template` | `EditVersionSchemeForm` (free text) | none, not currently enforced |
+| `committee_key` | `safe.CommitteeKey` | must be set |
+| `created` | set at creation | in the past |
+| `created_by` | none | ASF UID format |
 
 ### Release validation
 
diff --git a/atr/models/safe.py b/atr/models/safe.py
index 9a2c5566..12cbeba2 100644
--- a/atr/models/safe.py
+++ b/atr/models/safe.py
@@ -215,6 +215,10 @@ class ProjectKey(SafeType):
     def _valid_chars(cls) -> frozenset[str]:
         return _PROJECT_CHARS
 
+    def _additional_validations(self, value: str) -> None:
+        if value != value.lower():
+            raise ValueError("Project key must be lowercase")
+
 
 class ReleaseKey(Alphanumeric):
     """A release name composed from a validated ProjectKey and VersionKey."""
diff --git a/atr/post/projects.py b/atr/post/projects.py
index 87d85f45..ac1c7600 100644
--- a/atr/post/projects.py
+++ b/atr/post/projects.py
@@ -47,7 +47,7 @@ async def add_project(
     URL: /project/add/<committee_key>
     """
     display_name = project_form.display_name
-    label = project_form.label
+    project_key = project_form.key
 
     if committee_key != project_form.committee_key:
         raise ValueError(f"Invalid committee key: {committee_key}")
@@ -55,14 +55,14 @@ async def add_project(
     async with storage.write(session) as write:
         wacm = await write.as_project_committee_member(safe.ProjectKey(str(committee_key)))
         try:
-            await wacm.project.create(committee_key, display_name, label)
+            await wacm.project.create(committee_key, display_name, project_key)
         except storage.AccessError as e:
             return await session.redirect(
                 get.projects.add_project, committee_key=str(committee_key), error=f"Error adding project: {e}"
             )
 
     return await session.redirect(
-        get.projects.view, project_key=label, success=f"Project '{display_name}' added successfully."
+        get.projects.view, project_key=str(project_key), success=f"Project '{display_name}' added successfully."
     )
 
 
diff --git a/atr/shared/projects.py b/atr/shared/projects.py
index b9b67593..50919351 100644
--- a/atr/shared/projects.py
+++ b/atr/shared/projects.py
@@ -41,37 +41,36 @@ type ADD_LANGUAGE = Literal["add_language"]
 type REMOVE_LANGUAGE = Literal["remove_language"]
 
 
+def _strip_whitespace(value: object) -> object:
+    return value.strip() if isinstance(value, str) else value
+
+
 class AddProjectForm(form.Form):
     committee_key: safe.CommitteeKey = form.label("Committee key", widget=form.Widget.HIDDEN)
-    committee_key_display: str = form.label(description="Commitee key", widget=form.Widget.STATIC, default="")
+    committee_key_display: str = form.label(description="Committee key", widget=form.Widget.STATIC, default="")
     display_name: str = form.label(
         "Project name",
         'For example, "Apache Example" or "Apache Example Components". '
         'You must start with "Apache " and you must use title case.',
     )
-    label: str = form.label(
+    key: Annotated[safe.ProjectKey, pydantic.BeforeValidator(_strip_whitespace)] = form.label(
         "Project key",
         'For example, "example" or "example-components". '
         "You must start with your committee key (above), and you must use lower case.",
     )
 
-    @pydantic.model_validator(mode="after")
-    def validate_fields(self) -> AddProjectForm:
-        committee_key = str(self.committee_key)
-        display_name = self.display_name.strip()
-        label = self.label.strip()
-
+    @pydantic.field_validator("display_name", mode="before")
+    @classmethod
+    def validate_display_name(cls, val: str) -> str:
+        display_name = val.strip()
         # Normalise spaces in the display name
         display_name = re.sub(r"  +", " ", display_name)
 
-        # We must use object.__setattr__ to avoid calling the model validator again
-        object.__setattr__(self, "display_name", display_name)
+        display_name_words = display_name.split(" ")
 
         # Validate display name starts with "Apache"
-        display_name_words = display_name.split(" ")
         if display_name_words[0] != "Apache":
             raise ValueError("The first word in the name must be 'Apache'.")
-
         # Validate display name has at least two words
         if not display_name_words[1:]:
             raise ValueError("Name must have at least two words.")
@@ -94,19 +93,18 @@ class AddProjectForm(form.Form):
         if not display_name.replace(" ", "").replace(".", "").replace("+", "").isalnum():
             raise ValueError("Name must be alphanumeric and may include spaces or dots or plus signs.")
 
-        # Validate label starts with committee name
-        if not (label.startswith(committee_key + "-") or (label == committee_key)):
-            raise ValueError(f"Key must be '{committee_key}' or start with '{committee_key}-'.")
+        return display_name
 
-        # Validate label is lowercase
-        if not label.islower():
-            raise ValueError("Key must be all lower case.")
+    @pydantic.field_validator("key", mode="after")
+    @classmethod
+    def validate_fields(cls, val: safe.ProjectKey, info: pydantic.ValidationInfo) -> safe.ProjectKey:
+        committee_key = str(info.data.get("committee_key"))
+        key = str(val)
 
-        # Validate label is alphanumeric with hyphens
-        if not label.replace("-", "").isalnum():
-            raise ValueError("Key must be alphanumeric and may include hyphens.")
+        if not (key.startswith(committee_key + "-") or (key == committee_key)):
+            raise ValueError(f"Key must be '{committee_key}' or start with '{committee_key}-'.")
 
-        return self
+        return val
 
 
 class ComposePolicyForm(form.Form):
@@ -266,6 +264,32 @@ class EditVersionSchemeForm(form.Form):
         "Optional naming hint for source branches per cycle. Not currently enforced.",
     )
 
+    @pydantic.field_validator("version_pattern", mode="before")
+    @classmethod
+    def validate_version_scheme(cls, val: str) -> str:
+        version_pattern = val.strip()
+        if version_pattern:
+            try:
+                re.compile(version_pattern)
+            except re.error as e:
+                raise ValueError(f"Version pattern is not a valid regex: {e}")
+        return val
+
+    @pydantic.field_validator("cycle_match", mode="before")
+    @classmethod
+    def validate_cycle_match(cls, val: str) -> str:
+        cycle_match = val.strip()
+        if cycle_match:
+            try:
+                compiled = re.compile(cycle_match)
+            except re.error as e:
+                raise ValueError(f"Cycle match is not a valid regex: {e}")
+            # capture group 1 is the cycle name
+            if compiled.groups < 1:
+                raise ValueError("Cycle match must contain at least one capture group for the cycle name.")
+
+        return val
+
 
 class FinishPolicyForm(form.Form):
     variant: FINISH = form.value(FINISH)
diff --git a/atr/storage/writers/project.py b/atr/storage/writers/project.py
index 66649089..51a3f3a2 100644
--- a/atr/storage/writers/project.py
+++ b/atr/storage/writers/project.py
@@ -150,9 +150,9 @@ class CommitteeMember(CommitteeParticipant):
             return True
         return False
 
-    async def create(self, committee_key: safe.CommitteeKey, display_name: str, label: str) -> None:
+    async def create(self, committee_key: safe.CommitteeKey, display_name: str, project_key: safe.ProjectKey) -> None:
         try:
-            await self._build_and_add_project_no_commit(committee_key, display_name, label)
+            await self._build_and_add_project_no_commit(committee_key, display_name, project_key)
             await self.__data.commit()
         except sqlalchemy.exc.IntegrityError as e:
             if (
@@ -160,22 +160,22 @@ class CommitteeMember(CommitteeParticipant):
                 and (e.orig.sqlite_errorcode == sqlite3.SQLITE_CONSTRAINT_PRIMARYKEY)
                 and ("project.key" in str(e.orig))
             ):
-                raise storage.AccessError(f"Project {label} already exists", status=409)
+                raise storage.AccessError(f"Project {project_key} already exists", status=409)
             raise
         self.__write_as.append_to_audit_log(
             asf_uid=self.__asf_uid,
             committee_key=str(committee_key),
-            project_key=label,
+            project_key=str(project_key),
         )
 
     async def _build_and_add_project_no_commit(
         self,
         committee_key: safe.CommitteeKey,
         display_name: str,
-        label: str,
+        project_key: safe.ProjectKey,
     ) -> sql.Project:
         super_project = None
-        # TODO: Do we need to do any additional validation on the string value?
+        key = str(project_key)
         # Get the base project to derive from
         # We're allowing derivation from a retired project here
         # TODO: Should we disallow this instead?
@@ -183,17 +183,17 @@ class CommitteeMember(CommitteeParticipant):
             committee_key=str(committee_key), _committee=True, _release_policy=True
         ).all()
         for committee_project in committee_projects:
-            if label.startswith(str(committee_project.key) + "-"):
+            if key.startswith(str(committee_project.key) + "-"):
                 if (super_project is None) or (len(str(super_project.key)) < len(str(committee_project.key))):
                     super_project = committee_project
 
         # Check whether the project already exists
-        if await self.__data.project(key=label).get():
-            raise storage.AccessError(f"Project {label} already exists", status=409)
+        if await self.__data.project(key=key).get():
+            raise storage.AccessError(f"Project {key} already exists", status=409)
 
         now = datetime.datetime.now(datetime.UTC)
         project = sql.Project(
-            key=label,
+            key=key,
             name=display_name,
             status=sql.ProjectStatus.ACTIVE,
             super_project_key=super_project.key if super_project else None,
@@ -404,7 +404,7 @@ class CommitteeMember(CommitteeParticipant):
         if (args.project is None) or (args.project.name is None) or (not args.project.name.strip()):
             raise ValueError(f"Project '{args.project_key}' does not exist; project.name is required to create it")
         project = await self._build_and_add_project_no_commit(
-            args.committee_key, display_name=args.project.name, label=str(args.project_key)
+            args.committee_key, display_name=args.project.name, project_key=args.project_key
         )
         return project, True
 
diff --git a/atr/validate.py b/atr/validate.py
index 1b8c70b9..8efd1fa1 100644
--- a/atr/validate.py
+++ b/atr/validate.py
@@ -17,8 +17,9 @@
 
 import asyncio
 import datetime
+import re
 from collections.abc import AsyncGenerator, Callable, Generator, Iterable, Sequence
-from typing import NamedTuple, TypeVar
+from typing import Final, NamedTuple, TypeVar
 
 import atr.db as db
 import atr.models.safe as safe
@@ -50,6 +51,9 @@ type ReleaseAnnotatedDivergences = Callable[[sql.Release], AnnotatedDivergences]
 
 T = TypeVar("T")
 
+# Same ASF UID character set as principal.py
+_ASF_UID_PATTERN: Final = re.compile(r"^[-_a-z0-9]+$")
+
 if True:
 
     def committee_components(
@@ -93,7 +97,11 @@ def committee(c: sql.Committee) -> AnnotatedDivergences:
     """Check that a committee is valid."""
 
     yield from committee_child_committees(c)
+    yield from committee_committers(c)
     yield from committee_full_name(c)
+    yield from committee_key(c)
+    yield from committee_members(c)
+    yield from committee_release_managers(c)
 
 
 @committee_components("Committee.child_committees")
@@ -105,6 +113,15 @@ def committee_child_committees(c: sql.Committee) -> Divergences:
     yield from divergences(expected, actual)
 
 
+@committee_components("Committee.committers")
+def committee_committers(c: sql.Committee) -> Divergences:
+    def okay(uid: str) -> bool:
+        return bool(_ASF_UID_PATTERN.match(uid))
+
+    for uid in c.committers:
+        yield from divergences_predicate(okay, "value to be an ASF UID", uid)
+
+
 @committee_components("Committee.name")
 def committee_full_name(c: sql.Committee) -> Divergences:
     """Validate the Committee.name value."""
@@ -139,6 +156,40 @@ def committee_full_name(c: sql.Committee) -> Divergences:
     )
 
 
+@committee_components("Committee.key")
+def committee_key(c: sql.Committee) -> Divergences:
+    def okay(key: str) -> bool:
+        try:
+            safe.CommitteeKey(key)
+        except ValueError:
+            return False
+        return True
+
+    yield from divergences_predicate(okay, "value to be a valid committee key", c.key)
+
+
+@committee_components("Committee.committee_members")
+def committee_members(c: sql.Committee) -> Divergences:
+    """Check that every committee_members entry looks like an ASF UID."""
+
+    def okay(uid: str) -> bool:
+        return bool(_ASF_UID_PATTERN.match(uid))
+
+    for uid in c.committee_members:
+        yield from divergences_predicate(okay, "value to be an ASF UID", uid)
+
+
+@committee_components("Committee.release_managers")
+def committee_release_managers(c: sql.Committee) -> Divergences:
+    """Check that every release_managers entry looks like an ASF UID."""
+
+    def okay(uid: str) -> bool:
+        return bool(_ASF_UID_PATTERN.match(uid))
+
+    for uid in c.release_managers:
+        yield from divergences_predicate(okay, "value to be an ASF UID", uid)
+
+
 def committees(cs: Iterable[sql.Committee]) -> AnnotatedDivergences:
     """Validate multiple committees."""
     for c in cs:
@@ -190,9 +241,13 @@ def project(p: sql.Project) -> AnnotatedDivergences:
     yield from project_category(p)
     yield from project_committee(p)
     yield from project_created(p)
+    yield from project_created_by(p)
+    yield from project_cycle_match(p)
     yield from project_full_name(p)
+    yield from project_key(p)
     yield from project_programming_languages(p)
     yield from project_release_policy(p)
+    yield from project_version_pattern(p)
 
 
 @project_components("Project.category")
@@ -234,6 +289,35 @@ def project_created(p: sql.Project) -> Divergences:
     yield from divergences_predicate(predicate, expected, p.created)
 
 
+@project_components("Project.created_by")
+def project_created_by(p: sql.Project) -> Divergences:
+    """Check that created_by, if set, looks like an ASF UID."""
+
+    def okay(uid: str | None) -> bool:
+        if not uid:
+            return True
+        return bool(_ASF_UID_PATTERN.match(uid))
+
+    yield from divergences_predicate(okay, "value to be an ASF UID or unset", p.created_by)
+
+
+@project_components("Project.cycle_match")
+def project_cycle_match(p: sql.Project) -> Divergences:
+    """Check that cycle_match, if set, is a regex with at least one capture group."""
+
+    def okay(pattern: str | None) -> bool:
+        if not pattern:
+            return True
+        try:
+            compiled = re.compile(pattern)
+        except re.error:
+            return False
+        # Capture group 1 carries the cycle name, so we need at least one
+        return compiled.groups >= 1
+
+    yield from divergences_predicate(okay, "value to be a regex with a capture group, or unset", p.cycle_match)
+
+
 @project_components("Project.name")
 def project_full_name(p: sql.Project) -> Divergences:
     """Check that the project full_name is present and starts with 'Apache '."""
@@ -245,6 +329,20 @@ def project_full_name(p: sql.Project) -> Divergences:
     yield from divergences_predicate(okay, expected, p.name)
 
 
+@project_components("Project.key")
+def project_key(p: sql.Project) -> Divergences:
+    """Check that the project key is well-formed."""
+
+    def okay(key: str) -> bool:
+        try:
+            safe.ProjectKey(key)
+        except ValueError:
+            return False
+        return True
+
+    yield from divergences_predicate(okay, "value to be a valid project key", p.key)
+
+
 @project_components("Project.programming_languages")
 def project_programming_languages(p: sql.Project) -> Divergences:
     """Check that programming_languages uses 'label, label' syntax without colons."""
@@ -270,6 +368,22 @@ def project_release_policy(p: sql.Project) -> Divergences:
     yield from divergences(expected, actual)
 
 
+@project_components("Project.version_pattern")
+def project_version_pattern(p: sql.Project) -> Divergences:
+    """Check that version_pattern, if set, is a compilable regex."""
+
+    def okay(pattern: str | None) -> bool:
+        if not pattern:
+            return True
+        try:
+            re.compile(pattern)
+        except re.error:
+            return False
+        return True
+
+    yield from divergences_predicate(okay, "value to be a valid regex or unset", p.version_pattern)
+
+
 def projects(ps: Iterable[sql.Project]) -> AnnotatedDivergences:
     """Validate multiple projects."""
     for p in ps:
diff --git a/tests/unit/test_policy_writer.py b/tests/unit/test_policy_writer.py
index c75f87c9..917d9b85 100644
--- a/tests/unit/test_policy_writer.py
+++ b/tests/unit/test_policy_writer.py
@@ -19,6 +19,7 @@ import datetime
 import unittest.mock as mock
 from types import SimpleNamespace
 
+import pydantic
 import pytest
 
 import atr.models.safe as safe
@@ -259,9 +260,9 @@ async def test_edit_version_scheme_rejects_invalid_cycle_match_regex():
     project = _project()
     data = _MockData(project=project)
     writer = _make_committee_member(data)
-    form = _version_scheme_form(cycle_match="(unclosed")
 
-    with pytest.raises(ValueError, match="Invalid cycle_match regex"):
+    with pytest.raises(pydantic.ValidationError, match="Cycle match is not a valid regex"):
+        form = _version_scheme_form(cycle_match="(unclosed")
         await writer.edit_version_scheme(form)
 
 
@@ -269,9 +270,9 @@ async def test_edit_version_scheme_rejects_cycle_match_without_capture_group():
     project = _project()
     data = _MockData(project=project)
     writer = _make_committee_member(data)
-    form = _version_scheme_form(cycle_match=r"^\d+\.\d+\.\d+$")
 
-    with pytest.raises(ValueError, match="at least one capture group"):
+    with pytest.raises(pydantic.ValidationError, match="at least one capture group"):
+        form = _version_scheme_form(cycle_match=r"^\d+\.\d+\.\d+$")
         await writer.edit_version_scheme(form)
 
 
@@ -279,9 +280,9 @@ async def test_edit_version_scheme_rejects_invalid_version_pattern_regex():
     project = _project()
     data = _MockData(project=project)
     writer = _make_committee_member(data)
-    form = _version_scheme_form(version_pattern="(unclosed")
 
-    with pytest.raises(ValueError, match="Invalid version_pattern regex"):
+    with pytest.raises(pydantic.ValidationError, match="Version pattern is not a valid regex"):
+        form = _version_scheme_form(version_pattern="(unclosed")
         await writer.edit_version_scheme(form)
 
 


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org