You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@magpie.apache.org by po...@apache.org on 2026/06/30 23:59:37 UTC
(magpie) branch main updated: feat(validator): add Prerequisites sub-field consistency check for tool READMEs (#653)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/magpie.git
The following commit(s) were added to refs/heads/main by this push:
new 7e38106f feat(validator): add Prerequisites sub-field consistency check for tool READMEs (#653)
7e38106f is described below
commit 7e38106f6138eb049b358f852fb3731db4cee6a3
Author: Justin Mclean <ju...@classsoftware.com>
AuthorDate: Wed Jul 1 09:59:33 2026 +1000
feat(validator): add Prerequisites sub-field consistency check for tool READMEs (#653)
Normalize tools/security-tracker-stats-dashboard/README.md to the standard
four-label format (**Runtime:** / **CLIs:** / **Credentials / auth:** /
**Network:**) used by every other tool README.
Add TOOL_PREREQUISITES_FIELDS_CATEGORY as a HARD validator check verifying each
tool README's Prerequisites section carries either the four standard bold
sub-field labels or the accepted delegation shorthand
(**CLIs / credentials / network:** Provided by ...) for pure-contract tools.
Update five existing TestValidateTools fixtures that used bare placeholder
bullets to use the compliant four-field format via the shared _VALID_PREREQS
constant; add seven new test cases covering the standard format, the delegation
pattern, three missing-field scenarios, and the delegation-exempts-Network case.
Generated-by: Claude (Opus 4.7)
---
tools/security-tracker-stats-dashboard/README.md | 20 ++--
.../src/skill_and_tool_validator/__init__.py | 55 ++++++++++-
.../tests/test_validator.py | 108 +++++++++++++++++++--
3 files changed, 165 insertions(+), 18 deletions(-)
diff --git a/tools/security-tracker-stats-dashboard/README.md b/tools/security-tracker-stats-dashboard/README.md
index 6e68822b..b3db2cb2 100644
--- a/tools/security-tracker-stats-dashboard/README.md
+++ b/tools/security-tracker-stats-dashboard/README.md
@@ -219,16 +219,16 @@ remaining charts still render.
## Prerequisites
-- `gh` authenticated with read access to `<tracker>` (and to
- `<upstream>` for PR metadata, when configured).
-- `python3` (3.9+).
-- `jq` (only used by the fetch scripts via gh's `--jq` flag).
-- Network access to `api.github.com` and (for viewing) Plotly's CDN.
-- Optional: `pyyaml`. When missing, `render.py` falls back to a
- bundled minimal YAML subset parser sufficient for
- `default-config.yaml` and typical overlays. To pin a clean PyYAML
- invocation, set `TRACKER_STATS_PY=uv-yaml` and the orchestrator
- runs every step under `uv run --with pyyaml`.
+- **Runtime:** Python 3.9+ (`python3`). Run each stage as `python3 fetch_events.py`, etc.
+ Optional: `pyyaml` — when missing, `render.py` uses a bundled minimal YAML subset parser
+ sufficient for `default-config.yaml` and typical overlays; set `TRACKER_STATS_PY=uv-yaml`
+ to pin clean PyYAML invocations via `uv run --with pyyaml`.
+- **CLIs:** `gh` (authenticated with read access to `<tracker>`, and to `<upstream>` when
+ PR metadata is enabled); `jq` (used by fetch scripts via `gh --jq`).
+- **Credentials / auth:** `gh auth status` must show a logged-in account with read access to
+ `<tracker>`. `<upstream>` access is required only when `pr_repo` is configured.
+- **Network:** `api.github.com` (REST) for event and PR fetch; Plotly CDN
+ (`cdn.plot.ly`) when viewing the generated HTML dashboards in a browser.
## Failure modes
diff --git a/tools/skill-and-tool-validator/src/skill_and_tool_validator/__init__.py b/tools/skill-and-tool-validator/src/skill_and_tool_validator/__init__.py
index fa5e162b..f67e72b0 100644
--- a/tools/skill-and-tool-validator/src/skill_and_tool_validator/__init__.py
+++ b/tools/skill-and-tool-validator/src/skill_and_tool_validator/__init__.py
@@ -109,6 +109,7 @@ MODES_DOC_PATH = Path("docs/modes.md")
TOOL_README_CATEGORY = "tool-readme"
TOOL_CAPABILITY_CATEGORY = "tool-capability"
TOOL_PREREQUISITES_CATEGORY = "tool-prerequisites"
+TOOL_PREREQUISITES_FIELDS_CATEGORY = "tool-prerequisites-fields"
# Matches the `**Capability:** <token>` line (tool capability =
# `contract:NAME` / `substrate:NAME`, multi-value `a + b`) regardless of
@@ -160,6 +161,24 @@ _ADAPTER_CONFIG_RE = re.compile(
re.MULTILINE,
)
+# Sub-field regexes for the standard four-line Prerequisites layout:
+# **Runtime:** ...
+# **CLIs:** ... (or **CLIs / credentials / network:** for delegation)
+# **Credentials / auth:** ...
+# **Network:** ...
+# Labels follow the `**LABEL:** value` convention (colon inside the closing
+# bold markers, e.g. `**Runtime:**`). Each pattern matches either that style
+# OR the less-common `**LABEL**:` style (colon outside) to stay robust.
+# The delegation pattern (`**CLIs / credentials / network:**`) is the accepted
+# short form for pure-contract tools that delegate all three to an adapter.
+_PREREQ_RUNTIME_RE = re.compile(r"\*\*Runtime:?\*\*\s*:?", re.MULTILINE)
+_PREREQ_CLIS_RE = re.compile(r"\*\*CLIs?:?\*\*\s*:?", re.MULTILINE)
+_PREREQ_CREDENTIALS_RE = re.compile(r"\*\*Credentials?(?:\s*/\s*auth)?:?\*\*\s*:?", re.MULTILINE)
+_PREREQ_NETWORK_RE = re.compile(r"\*\*Network:?\*\*\s*:?", re.MULTILINE)
+_PREREQ_DELEGATION_RE = re.compile(
+ r"\*\*CLIs\s*/\s*credentials\s*/\s*network:?\*\*\s*:?", re.MULTILINE | re.IGNORECASE
+)
+
# Optional `**Organization:** <org>` line in a tool README — declares that
# the tool belongs to / is the adapter for a specific organization (e.g.
# the ASF backends cve-tool-vulnogram, ponymail, apache-projects). Absent =
@@ -398,6 +417,7 @@ HARD_CATEGORIES: frozenset[str] = frozenset(
TOOL_README_CATEGORY,
TOOL_CAPABILITY_CATEGORY,
TOOL_PREREQUISITES_CATEGORY,
+ TOOL_PREREQUISITES_FIELDS_CATEGORY,
ORGANIZATION_CATEGORY,
CAPABILITY_SYNC_CATEGORY,
INJECTION_GUARD_CATEGORY,
@@ -1471,7 +1491,8 @@ def validate_tools(root: Path | None = None) -> Iterable[Violation]:
yield Violation(readme, None, f"cannot read README.md: {exc}")
continue
- if TOOL_PREREQUISITES_RE.search(text) is None:
+ prereq_match = TOOL_PREREQUISITES_RE.search(text)
+ if prereq_match is None:
yield Violation(
readme,
1,
@@ -1480,6 +1501,38 @@ def validate_tools(root: Path | None = None) -> Iterable[Violation]:
f"access up front (see tools/AGENTS.md)",
category=TOOL_PREREQUISITES_CATEGORY,
)
+ else:
+ # Validate sub-field structure within the Prerequisites section.
+ # Each section must declare **Runtime:**, **CLIs:**, **Credentials /
+ # auth:**, and **Network:** as bold bullet labels, OR use the
+ # accepted delegation shorthand **CLIs / credentials / network:**
+ # (for pure-contract tools that proxy all three to a concrete adapter).
+ next_heading = re.search(r"^##\s+", text[prereq_match.end() :], re.MULTILINE)
+ section_end = prereq_match.end() + next_heading.start() if next_heading else len(text)
+ section = text[prereq_match.start() : section_end]
+ has_delegation = bool(_PREREQ_DELEGATION_RE.search(section))
+ missing: list[str] = []
+ if not _PREREQ_RUNTIME_RE.search(section):
+ missing.append("**Runtime:**")
+ if not has_delegation:
+ if not _PREREQ_CLIS_RE.search(section):
+ missing.append("**CLIs:**")
+ if not _PREREQ_CREDENTIALS_RE.search(section):
+ missing.append("**Credentials / auth:**")
+ if not _PREREQ_NETWORK_RE.search(section):
+ missing.append("**Network:**")
+ if missing:
+ yield Violation(
+ readme,
+ text[: prereq_match.start()].count("\n") + 1,
+ f"tool '{tool_dir.name}' Prerequisites section missing required "
+ f"sub-field(s): {', '.join(missing)} — use bold labels "
+ f"(**Runtime:**, **CLIs:**, **Credentials / auth:**, **Network:**) "
+ f"or the delegation shorthand "
+ f"(**CLIs / credentials / network:** Provided by …) "
+ f"for pure-contract tools",
+ category=TOOL_PREREQUISITES_FIELDS_CATEGORY,
+ )
org_match = TOOL_ORGANIZATION_RE.search(text)
if org_match is not None:
diff --git a/tools/skill-and-tool-validator/tests/test_validator.py b/tools/skill-and-tool-validator/tests/test_validator.py
index ccce7f98..6bdcb205 100644
--- a/tools/skill-and-tool-validator/tests/test_validator.py
+++ b/tools/skill-and-tool-validator/tests/test_validator.py
@@ -2449,14 +2449,28 @@ class TestValidateAsfCoupling:
assert all(v.category != ASF_COUPLING_CATEGORY for v in violations)
+_VALID_PREREQS = (
+ "## Prerequisites\n\n"
+ "- **Runtime:** None.\n"
+ "- **CLIs:** None.\n"
+ "- **Credentials / auth:** None.\n"
+ "- **Network:** None.\n"
+)
+
+_DELEGATION_PREREQS = (
+ "## Prerequisites\n\n"
+ "- **Runtime:** None — pure Markdown contract.\n"
+ "- **CLIs / credentials / network:** Provided by the concrete adapter.\n"
+)
+
+
class TestValidateTools:
def test_tool_with_valid_readme(self, tmp_path: Path) -> None:
root = _make_tools_root(tmp_path)
tool = root / "tools" / "foo"
tool.mkdir()
(tool / "README.md").write_text(
- "# tools/foo\n\n**Capability:** substrate:framework-dev\n\nFoo tool.\n\n"
- "## Prerequisites\n\n- Python 3.11+ via uv.\n"
+ "# tools/foo\n\n**Capability:** substrate:framework-dev\n\nFoo tool.\n\n" + _VALID_PREREQS
)
violations = list(validate_tools(root))
assert violations == []
@@ -2474,7 +2488,7 @@ class TestValidateTools:
tool = root / "tools" / "bare"
tool.mkdir()
(tool / "README.md").write_text(
- "# bare\n\nDescription only, no capability line.\n\n## Prerequisites\n\n- None.\n"
+ "# bare\n\nDescription only, no capability line.\n\n" + _VALID_PREREQS
)
violations = list(validate_tools(root))
assert len(violations) == 1
@@ -2494,7 +2508,7 @@ class TestValidateTools:
tool = root / "tools" / "dual"
tool.mkdir()
(tool / "README.md").write_text(
- "# dual\n\n**Capability:** contract:tracker + substrate:analytics\n\n## Prerequisites\n\n- None.\n"
+ "# dual\n\n**Capability:** contract:tracker + substrate:analytics\n\n" + _VALID_PREREQS
)
violations = list(validate_tools(root))
assert violations == []
@@ -2510,7 +2524,7 @@ class TestValidateTools:
"# tools/with-prose\n\n"
"**Capability:** substrate:framework-dev\n\n"
"Some prose that follows the capability line and should NOT be parsed as part of it.\n\n"
- "## Prerequisites\n\n- None.\n"
+ + _VALID_PREREQS
)
violations = list(validate_tools(root))
assert violations == []
@@ -2540,7 +2554,7 @@ class TestValidateTools:
tool.mkdir()
(tool / "README.md").write_text(
"# asf-backend\n\n**Capability:** substrate:framework-dev\n\n"
- "**Organization:** ASF\n\nAn ASF backend.\n\n## Prerequisites\n\n- None.\n"
+ "**Organization:** ASF\n\nAn ASF backend.\n\n" + _VALID_PREREQS
)
violations = list(validate_tools(root))
assert violations == []
@@ -2552,12 +2566,92 @@ class TestValidateTools:
tool.mkdir()
(tool / "README.md").write_text(
"# bogus-org\n\n**Capability:** substrate:framework-dev\n\n"
- "**Organization:** Nope\n\nA tool.\n\n## Prerequisites\n\n- None.\n"
+ "**Organization:** Nope\n\nA tool.\n\n" + _VALID_PREREQS
)
violations = [v for v in validate_tools(root) if v.category == "organization"]
assert len(violations) == 1
assert "'**Organization:** Nope'" in violations[0].message
+ def test_prerequisites_subfields_standard_format(self, tmp_path: Path) -> None:
+ root = _make_tools_root(tmp_path)
+ tool = root / "tools" / "standard"
+ tool.mkdir()
+ (tool / "README.md").write_text(
+ "# standard\n\n**Capability:** substrate:framework-dev\n\n" + _VALID_PREREQS
+ )
+ violations = [v for v in validate_tools(root) if v.category == "tool-prerequisites-fields"]
+ assert violations == []
+
+ def test_prerequisites_subfields_delegation_pattern(self, tmp_path: Path) -> None:
+ # A pure-contract tool may use the delegation shorthand for CLIs/credentials/network.
+ root = _make_tools_root(tmp_path)
+ tool = root / "tools" / "contract"
+ tool.mkdir()
+ (tool / "README.md").write_text(
+ "# contract\n\n**Capability:** contract:mail-source\n\n" + _DELEGATION_PREREQS
+ )
+ violations = [v for v in validate_tools(root) if v.category == "tool-prerequisites-fields"]
+ assert violations == []
+
+ def test_prerequisites_subfields_missing_runtime(self, tmp_path: Path) -> None:
+ root = _make_tools_root(tmp_path)
+ tool = root / "tools" / "no-runtime"
+ tool.mkdir()
+ (tool / "README.md").write_text(
+ "# no-runtime\n\n**Capability:** substrate:framework-dev\n\n"
+ "## Prerequisites\n\n"
+ "- **CLIs:** None.\n"
+ "- **Credentials / auth:** None.\n"
+ "- **Network:** None.\n"
+ )
+ violations = [v for v in validate_tools(root) if v.category == "tool-prerequisites-fields"]
+ assert len(violations) == 1
+ assert "**Runtime:**" in violations[0].message
+
+ def test_prerequisites_subfields_missing_network(self, tmp_path: Path) -> None:
+ root = _make_tools_root(tmp_path)
+ tool = root / "tools" / "no-network"
+ tool.mkdir()
+ (tool / "README.md").write_text(
+ "# no-network\n\n**Capability:** substrate:framework-dev\n\n"
+ "## Prerequisites\n\n"
+ "- **Runtime:** Python 3.11+.\n"
+ "- **CLIs:** None.\n"
+ "- **Credentials / auth:** None.\n"
+ )
+ violations = [v for v in validate_tools(root) if v.category == "tool-prerequisites-fields"]
+ assert len(violations) == 1
+ assert "**Network:**" in violations[0].message
+
+ def test_prerequisites_subfields_missing_clis_and_credentials(self, tmp_path: Path) -> None:
+ root = _make_tools_root(tmp_path)
+ tool = root / "tools" / "no-clis"
+ tool.mkdir()
+ (tool / "README.md").write_text(
+ "# no-clis\n\n**Capability:** substrate:framework-dev\n\n"
+ "## Prerequisites\n\n"
+ "- **Runtime:** Python 3.11+.\n"
+ "- **Network:** api.github.com.\n"
+ )
+ violations = [v for v in validate_tools(root) if v.category == "tool-prerequisites-fields"]
+ assert len(violations) == 1
+ assert "**CLIs:**" in violations[0].message
+ assert "**Credentials / auth:**" in violations[0].message
+
+ def test_prerequisites_subfields_delegation_no_network_required(self, tmp_path: Path) -> None:
+ # Delegation pattern covers CLIs + credentials + network; no separate Network: needed.
+ root = _make_tools_root(tmp_path)
+ tool = root / "tools" / "delegate"
+ tool.mkdir()
+ (tool / "README.md").write_text(
+ "# delegate\n\n**Capability:** contract:cve-authority\n\n"
+ "## Prerequisites\n\n"
+ "- **Runtime:** None — Markdown contract spec.\n"
+ "- **CLIs / credentials / network:** Provided by the concrete adapter.\n"
+ )
+ violations = [v for v in validate_tools(root) if v.category == "tool-prerequisites-fields"]
+ assert violations == []
+
class TestValidateAdapterAuthoring:
"""Tests for the SOFT adapter authoring smoke check (aspect #11)."""