You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tooling.apache.org by ar...@apache.org on 2026/05/21 13:58:18 UTC
(tooling-trusted-releases) branch arm updated: #470 - add validation rules for new fields, update form validation for add project to use field validators
This is an automated email from the ASF dual-hosted git repository.
alitheg pushed a commit to branch arm
in repository https://gitbox.apache.org/repos/asf/tooling-trusted-releases.git
The following commit(s) were added to refs/heads/arm by this push:
new 8372b3fe #470 - add validation rules for new fields, update form validation for add project to use field validators
8372b3fe is described below
commit 8372b3fe415e22a3562d76b853cdb73b8465a928
Author: Alastair McFarlane <ar...@apache.org>
AuthorDate: Thu May 21 14:58:00 2026 +0100
#470 - add validation rules for new fields, update form validation for add project to use field validators
---
atr/docs/input-validation.md | 43 ++++++++++++++-
atr/models/safe.py | 4 ++
atr/post/projects.py | 6 +-
atr/shared/projects.py | 68 +++++++++++++++--------
atr/storage/writers/project.py | 22 ++++----
atr/validate.py | 116 ++++++++++++++++++++++++++++++++++++++-
tests/unit/test_policy_writer.py | 13 +++--
7 files changed, 228 insertions(+), 44 deletions(-)
diff --git a/atr/docs/input-validation.md b/atr/docs/input-validation.md
index db4a5ed3..e169bbb1 100644
--- a/atr/docs/input-validation.md
+++ b/atr/docs/input-validation.md
@@ -166,21 +166,62 @@ Beyond input validation, ATR performs data integrity validation on database reco
### Committee validation
+Committee records are not created through a form. They are synced from LDAP and Whimsy by [`_update_committees`](/ref/atr/datasources/apache.py:_update_committees), so their fields have no Pydantic input layer, and the data integrity validators are the main place each field is confirmed.
+
The [`committee`](/ref/atr/validate.py:committee) function checks:
+* `key` must be a valid committee key (the [`safe.CommitteeKey`](/ref/atr/models/safe.py:CommitteeKey) character set)
+* `name` must be set, trimmed, and not prefixed with "Apache "
+* `committee_members`, `committers`, and `release_managers` entries must each look like an ASF UID
* `child_committees` must be empty (not used)
-* `full_name` must be set, trimmed, and not prefixed with "Apache "
+
+Per-field coverage:
+
+| Field | Input layer | Data integrity |
+| --- | --- | --- |
+| `key` | none (LDAP project name) | valid committee key |
+| `name` | none (Whimsy) | set, trimmed, no "Apache " prefix |
+| `is_podling` | set during sync | type-enforced (bool) |
+| `parent_committee_key` | none | foreign key |
+| `committee_members` | none (LDAP) | ASF UID format |
+| `committers` | none (LDAP) | ASF UID format |
+| `release_managers` | none | ASF UID format |
+| `child_committees` | none | must be empty |
### Project validation
+Projects do have an input layer. [`AddProjectForm`](/ref/atr/shared/projects.py:AddProjectForm) validates the key and display name at creation, and [`EditVersionSchemeForm`](/ref/atr/shared/projects.py:EditVersionSchemeForm) rejects a `version_pattern` or `cycle_match` that is not a valid regex (and a `cycle_match` with no capture group). The data integrity validators mirror these and cover fields that have no form.
+
The [`project`](/ref/atr/validate.py:project) function checks:
+* `key` must be a valid project key (the [`safe.ProjectKey`](/ref/atr/models/safe.py:ProjectKey) character set)
* `category` must use comma-separated labels without colons
* `committee_key` must be set (project must be linked to a committee)
* `created` timestamp must be in the past
+* `created_by`, if set, must look like an ASF UID
+* `cycle_match`, if set, must be a regex with at least one capture group
* `full_name` must be set and start with "Apache "
* `programming_languages` must use comma-separated labels without colons
* `release_policy_id` must be None (not used)
+* `version_pattern`, if set, must be a compilable regex
+
+Per-field coverage:
+
+| Field | Input layer | Data integrity |
+| --- | --- | --- |
+| `key` | `safe.ProjectKey` (charset, lowercase) + `AddProjectForm` (committee-prefixed) | valid project key |
+| `name` | `AddProjectForm` (Apache prefix, case rules) | set, "Apache " prefix |
+| `status` | none (enum) | type-enforced (enum) |
+| `description` | `EditMetadataForm` (free text) | none |
+| `category` | `AddCategoryForm` (free text) | comma-separated, no colons |
+| `programming_languages` | `AddLanguageForm` (free text) | comma-separated, no colons |
+| `version_method` | `EditVersionSchemeForm` (enum) | type-enforced (enum) |
+| `version_pattern` | `EditVersionSchemeForm` (regex validity) | compilable regex |
+| `cycle_match` | `EditVersionSchemeForm` (regex and capture group) | regex with capture group |
+| `branch_template` | `EditVersionSchemeForm` (free text) | none, not currently enforced |
+| `committee_key` | `safe.CommitteeKey` | must be set |
+| `created` | set at creation | in the past |
+| `created_by` | none | ASF UID format |
### Release validation
diff --git a/atr/models/safe.py b/atr/models/safe.py
index 9a2c5566..12cbeba2 100644
--- a/atr/models/safe.py
+++ b/atr/models/safe.py
@@ -215,6 +215,10 @@ class ProjectKey(SafeType):
def _valid_chars(cls) -> frozenset[str]:
return _PROJECT_CHARS
+ def _additional_validations(self, value: str) -> None:
+ if value != value.lower():
+ raise ValueError("Project key must be lowercase")
+
class ReleaseKey(Alphanumeric):
"""A release name composed from a validated ProjectKey and VersionKey."""
diff --git a/atr/post/projects.py b/atr/post/projects.py
index 87d85f45..ac1c7600 100644
--- a/atr/post/projects.py
+++ b/atr/post/projects.py
@@ -47,7 +47,7 @@ async def add_project(
URL: /project/add/<committee_key>
"""
display_name = project_form.display_name
- label = project_form.label
+ project_key = project_form.key
if committee_key != project_form.committee_key:
raise ValueError(f"Invalid committee key: {committee_key}")
@@ -55,14 +55,14 @@ async def add_project(
async with storage.write(session) as write:
wacm = await write.as_project_committee_member(safe.ProjectKey(str(committee_key)))
try:
- await wacm.project.create(committee_key, display_name, label)
+ await wacm.project.create(committee_key, display_name, project_key)
except storage.AccessError as e:
return await session.redirect(
get.projects.add_project, committee_key=str(committee_key), error=f"Error adding project: {e}"
)
return await session.redirect(
- get.projects.view, project_key=label, success=f"Project '{display_name}' added successfully."
+ get.projects.view, project_key=str(project_key), success=f"Project '{display_name}' added successfully."
)
diff --git a/atr/shared/projects.py b/atr/shared/projects.py
index b9b67593..50919351 100644
--- a/atr/shared/projects.py
+++ b/atr/shared/projects.py
@@ -41,37 +41,36 @@ type ADD_LANGUAGE = Literal["add_language"]
type REMOVE_LANGUAGE = Literal["remove_language"]
+def _strip_whitespace(value: object) -> object:
+ return value.strip() if isinstance(value, str) else value
+
+
class AddProjectForm(form.Form):
committee_key: safe.CommitteeKey = form.label("Committee key", widget=form.Widget.HIDDEN)
- committee_key_display: str = form.label(description="Commitee key", widget=form.Widget.STATIC, default="")
+ committee_key_display: str = form.label(description="Committee key", widget=form.Widget.STATIC, default="")
display_name: str = form.label(
"Project name",
'For example, "Apache Example" or "Apache Example Components". '
'You must start with "Apache " and you must use title case.',
)
- label: str = form.label(
+ key: Annotated[safe.ProjectKey, pydantic.BeforeValidator(_strip_whitespace)] = form.label(
"Project key",
'For example, "example" or "example-components". '
"You must start with your committee key (above), and you must use lower case.",
)
- @pydantic.model_validator(mode="after")
- def validate_fields(self) -> AddProjectForm:
- committee_key = str(self.committee_key)
- display_name = self.display_name.strip()
- label = self.label.strip()
-
+ @pydantic.field_validator("display_name", mode="before")
+ @classmethod
+ def validate_display_name(cls, val: str) -> str:
+ display_name = val.strip()
# Normalise spaces in the display name
display_name = re.sub(r" +", " ", display_name)
- # We must use object.__setattr__ to avoid calling the model validator again
- object.__setattr__(self, "display_name", display_name)
+ display_name_words = display_name.split(" ")
# Validate display name starts with "Apache"
- display_name_words = display_name.split(" ")
if display_name_words[0] != "Apache":
raise ValueError("The first word in the name must be 'Apache'.")
-
# Validate display name has at least two words
if not display_name_words[1:]:
raise ValueError("Name must have at least two words.")
@@ -94,19 +93,18 @@ class AddProjectForm(form.Form):
if not display_name.replace(" ", "").replace(".", "").replace("+", "").isalnum():
raise ValueError("Name must be alphanumeric and may include spaces or dots or plus signs.")
- # Validate label starts with committee name
- if not (label.startswith(committee_key + "-") or (label == committee_key)):
- raise ValueError(f"Key must be '{committee_key}' or start with '{committee_key}-'.")
+ return display_name
- # Validate label is lowercase
- if not label.islower():
- raise ValueError("Key must be all lower case.")
+ @pydantic.field_validator("key", mode="after")
+ @classmethod
+ def validate_fields(cls, val: safe.ProjectKey, info: pydantic.ValidationInfo) -> safe.ProjectKey:
+ committee_key = str(info.data.get("committee_key"))
+ key = str(val)
- # Validate label is alphanumeric with hyphens
- if not label.replace("-", "").isalnum():
- raise ValueError("Key must be alphanumeric and may include hyphens.")
+ if not (key.startswith(committee_key + "-") or (key == committee_key)):
+ raise ValueError(f"Key must be '{committee_key}' or start with '{committee_key}-'.")
- return self
+ return val
class ComposePolicyForm(form.Form):
@@ -266,6 +264,32 @@ class EditVersionSchemeForm(form.Form):
"Optional naming hint for source branches per cycle. Not currently enforced.",
)
+ @pydantic.field_validator("version_pattern", mode="before")
+ @classmethod
+ def validate_version_scheme(cls, val: str) -> str:
+ version_pattern = val.strip()
+ if version_pattern:
+ try:
+ re.compile(version_pattern)
+ except re.error as e:
+ raise ValueError(f"Version pattern is not a valid regex: {e}")
+ return val
+
+ @pydantic.field_validator("cycle_match", mode="before")
+ @classmethod
+ def validate_cycle_match(cls, val: str) -> str:
+ cycle_match = val.strip()
+ if cycle_match:
+ try:
+ compiled = re.compile(cycle_match)
+ except re.error as e:
+ raise ValueError(f"Cycle match is not a valid regex: {e}")
+ # capture group 1 is the cycle name
+ if compiled.groups < 1:
+ raise ValueError("Cycle match must contain at least one capture group for the cycle name.")
+
+ return val
+
class FinishPolicyForm(form.Form):
variant: FINISH = form.value(FINISH)
diff --git a/atr/storage/writers/project.py b/atr/storage/writers/project.py
index 66649089..51a3f3a2 100644
--- a/atr/storage/writers/project.py
+++ b/atr/storage/writers/project.py
@@ -150,9 +150,9 @@ class CommitteeMember(CommitteeParticipant):
return True
return False
- async def create(self, committee_key: safe.CommitteeKey, display_name: str, label: str) -> None:
+ async def create(self, committee_key: safe.CommitteeKey, display_name: str, project_key: safe.ProjectKey) -> None:
try:
- await self._build_and_add_project_no_commit(committee_key, display_name, label)
+ await self._build_and_add_project_no_commit(committee_key, display_name, project_key)
await self.__data.commit()
except sqlalchemy.exc.IntegrityError as e:
if (
@@ -160,22 +160,22 @@ class CommitteeMember(CommitteeParticipant):
and (e.orig.sqlite_errorcode == sqlite3.SQLITE_CONSTRAINT_PRIMARYKEY)
and ("project.key" in str(e.orig))
):
- raise storage.AccessError(f"Project {label} already exists", status=409)
+ raise storage.AccessError(f"Project {project_key} already exists", status=409)
raise
self.__write_as.append_to_audit_log(
asf_uid=self.__asf_uid,
committee_key=str(committee_key),
- project_key=label,
+ project_key=str(project_key),
)
async def _build_and_add_project_no_commit(
self,
committee_key: safe.CommitteeKey,
display_name: str,
- label: str,
+ project_key: safe.ProjectKey,
) -> sql.Project:
super_project = None
- # TODO: Do we need to do any additional validation on the string value?
+ key = str(project_key)
# Get the base project to derive from
# We're allowing derivation from a retired project here
# TODO: Should we disallow this instead?
@@ -183,17 +183,17 @@ class CommitteeMember(CommitteeParticipant):
committee_key=str(committee_key), _committee=True, _release_policy=True
).all()
for committee_project in committee_projects:
- if label.startswith(str(committee_project.key) + "-"):
+ if key.startswith(str(committee_project.key) + "-"):
if (super_project is None) or (len(str(super_project.key)) < len(str(committee_project.key))):
super_project = committee_project
# Check whether the project already exists
- if await self.__data.project(key=label).get():
- raise storage.AccessError(f"Project {label} already exists", status=409)
+ if await self.__data.project(key=key).get():
+ raise storage.AccessError(f"Project {key} already exists", status=409)
now = datetime.datetime.now(datetime.UTC)
project = sql.Project(
- key=label,
+ key=key,
name=display_name,
status=sql.ProjectStatus.ACTIVE,
super_project_key=super_project.key if super_project else None,
@@ -404,7 +404,7 @@ class CommitteeMember(CommitteeParticipant):
if (args.project is None) or (args.project.name is None) or (not args.project.name.strip()):
raise ValueError(f"Project '{args.project_key}' does not exist; project.name is required to create it")
project = await self._build_and_add_project_no_commit(
- args.committee_key, display_name=args.project.name, label=str(args.project_key)
+ args.committee_key, display_name=args.project.name, project_key=args.project_key
)
return project, True
diff --git a/atr/validate.py b/atr/validate.py
index 1b8c70b9..8efd1fa1 100644
--- a/atr/validate.py
+++ b/atr/validate.py
@@ -17,8 +17,9 @@
import asyncio
import datetime
+import re
from collections.abc import AsyncGenerator, Callable, Generator, Iterable, Sequence
-from typing import NamedTuple, TypeVar
+from typing import Final, NamedTuple, TypeVar
import atr.db as db
import atr.models.safe as safe
@@ -50,6 +51,9 @@ type ReleaseAnnotatedDivergences = Callable[[sql.Release], AnnotatedDivergences]
T = TypeVar("T")
+# Same ASF UID character set as principal.py
+_ASF_UID_PATTERN: Final = re.compile(r"^[-_a-z0-9]+$")
+
if True:
def committee_components(
@@ -93,7 +97,11 @@ def committee(c: sql.Committee) -> AnnotatedDivergences:
"""Check that a committee is valid."""
yield from committee_child_committees(c)
+ yield from committee_committers(c)
yield from committee_full_name(c)
+ yield from committee_key(c)
+ yield from committee_members(c)
+ yield from committee_release_managers(c)
@committee_components("Committee.child_committees")
@@ -105,6 +113,15 @@ def committee_child_committees(c: sql.Committee) -> Divergences:
yield from divergences(expected, actual)
+@committee_components("Committee.committers")
+def committee_committers(c: sql.Committee) -> Divergences:
+ def okay(uid: str) -> bool:
+ return bool(_ASF_UID_PATTERN.match(uid))
+
+ for uid in c.committers:
+ yield from divergences_predicate(okay, "value to be an ASF UID", uid)
+
+
@committee_components("Committee.name")
def committee_full_name(c: sql.Committee) -> Divergences:
"""Validate the Committee.name value."""
@@ -139,6 +156,40 @@ def committee_full_name(c: sql.Committee) -> Divergences:
)
+@committee_components("Committee.key")
+def committee_key(c: sql.Committee) -> Divergences:
+ def okay(key: str) -> bool:
+ try:
+ safe.CommitteeKey(key)
+ except ValueError:
+ return False
+ return True
+
+ yield from divergences_predicate(okay, "value to be a valid committee key", c.key)
+
+
+@committee_components("Committee.committee_members")
+def committee_members(c: sql.Committee) -> Divergences:
+ """Check that every committee_members entry looks like an ASF UID."""
+
+ def okay(uid: str) -> bool:
+ return bool(_ASF_UID_PATTERN.match(uid))
+
+ for uid in c.committee_members:
+ yield from divergences_predicate(okay, "value to be an ASF UID", uid)
+
+
+@committee_components("Committee.release_managers")
+def committee_release_managers(c: sql.Committee) -> Divergences:
+ """Check that every release_managers entry looks like an ASF UID."""
+
+ def okay(uid: str) -> bool:
+ return bool(_ASF_UID_PATTERN.match(uid))
+
+ for uid in c.release_managers:
+ yield from divergences_predicate(okay, "value to be an ASF UID", uid)
+
+
def committees(cs: Iterable[sql.Committee]) -> AnnotatedDivergences:
"""Validate multiple committees."""
for c in cs:
@@ -190,9 +241,13 @@ def project(p: sql.Project) -> AnnotatedDivergences:
yield from project_category(p)
yield from project_committee(p)
yield from project_created(p)
+ yield from project_created_by(p)
+ yield from project_cycle_match(p)
yield from project_full_name(p)
+ yield from project_key(p)
yield from project_programming_languages(p)
yield from project_release_policy(p)
+ yield from project_version_pattern(p)
@project_components("Project.category")
@@ -234,6 +289,35 @@ def project_created(p: sql.Project) -> Divergences:
yield from divergences_predicate(predicate, expected, p.created)
+@project_components("Project.created_by")
+def project_created_by(p: sql.Project) -> Divergences:
+ """Check that created_by, if set, looks like an ASF UID."""
+
+ def okay(uid: str | None) -> bool:
+ if not uid:
+ return True
+ return bool(_ASF_UID_PATTERN.match(uid))
+
+ yield from divergences_predicate(okay, "value to be an ASF UID or unset", p.created_by)
+
+
+@project_components("Project.cycle_match")
+def project_cycle_match(p: sql.Project) -> Divergences:
+ """Check that cycle_match, if set, is a regex with at least one capture group."""
+
+ def okay(pattern: str | None) -> bool:
+ if not pattern:
+ return True
+ try:
+ compiled = re.compile(pattern)
+ except re.error:
+ return False
+ # Capture group 1 carries the cycle name, so we need at least one
+ return compiled.groups >= 1
+
+ yield from divergences_predicate(okay, "value to be a regex with a capture group, or unset", p.cycle_match)
+
+
@project_components("Project.name")
def project_full_name(p: sql.Project) -> Divergences:
"""Check that the project full_name is present and starts with 'Apache '."""
@@ -245,6 +329,20 @@ def project_full_name(p: sql.Project) -> Divergences:
yield from divergences_predicate(okay, expected, p.name)
+@project_components("Project.key")
+def project_key(p: sql.Project) -> Divergences:
+ """Check that the project key is well-formed."""
+
+ def okay(key: str) -> bool:
+ try:
+ safe.ProjectKey(key)
+ except ValueError:
+ return False
+ return True
+
+ yield from divergences_predicate(okay, "value to be a valid project key", p.key)
+
+
@project_components("Project.programming_languages")
def project_programming_languages(p: sql.Project) -> Divergences:
"""Check that programming_languages uses 'label, label' syntax without colons."""
@@ -270,6 +368,22 @@ def project_release_policy(p: sql.Project) -> Divergences:
yield from divergences(expected, actual)
+@project_components("Project.version_pattern")
+def project_version_pattern(p: sql.Project) -> Divergences:
+ """Check that version_pattern, if set, is a compilable regex."""
+
+ def okay(pattern: str | None) -> bool:
+ if not pattern:
+ return True
+ try:
+ re.compile(pattern)
+ except re.error:
+ return False
+ return True
+
+ yield from divergences_predicate(okay, "value to be a valid regex or unset", p.version_pattern)
+
+
def projects(ps: Iterable[sql.Project]) -> AnnotatedDivergences:
"""Validate multiple projects."""
for p in ps:
diff --git a/tests/unit/test_policy_writer.py b/tests/unit/test_policy_writer.py
index c75f87c9..917d9b85 100644
--- a/tests/unit/test_policy_writer.py
+++ b/tests/unit/test_policy_writer.py
@@ -19,6 +19,7 @@ import datetime
import unittest.mock as mock
from types import SimpleNamespace
+import pydantic
import pytest
import atr.models.safe as safe
@@ -259,9 +260,9 @@ async def test_edit_version_scheme_rejects_invalid_cycle_match_regex():
project = _project()
data = _MockData(project=project)
writer = _make_committee_member(data)
- form = _version_scheme_form(cycle_match="(unclosed")
- with pytest.raises(ValueError, match="Invalid cycle_match regex"):
+ with pytest.raises(pydantic.ValidationError, match="Cycle match is not a valid regex"):
+ form = _version_scheme_form(cycle_match="(unclosed")
await writer.edit_version_scheme(form)
@@ -269,9 +270,9 @@ async def test_edit_version_scheme_rejects_cycle_match_without_capture_group():
project = _project()
data = _MockData(project=project)
writer = _make_committee_member(data)
- form = _version_scheme_form(cycle_match=r"^\d+\.\d+\.\d+$")
- with pytest.raises(ValueError, match="at least one capture group"):
+ with pytest.raises(pydantic.ValidationError, match="at least one capture group"):
+ form = _version_scheme_form(cycle_match=r"^\d+\.\d+\.\d+$")
await writer.edit_version_scheme(form)
@@ -279,9 +280,9 @@ async def test_edit_version_scheme_rejects_invalid_version_pattern_regex():
project = _project()
data = _MockData(project=project)
writer = _make_committee_member(data)
- form = _version_scheme_form(version_pattern="(unclosed")
- with pytest.raises(ValueError, match="Invalid version_pattern regex"):
+ with pytest.raises(pydantic.ValidationError, match="Version pattern is not a valid regex"):
+ form = _version_scheme_form(version_pattern="(unclosed")
await writer.edit_version_scheme(form)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org