You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@magpie.apache.org by po...@apache.org on 2026/07/03 13:56:50 UTC
(magpie) branch main updated: feat(validator): add capability taxonomy coverage check (#693)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/magpie.git
The following commit(s) were added to refs/heads/main by this push:
new feee2aa9 feat(validator): add capability taxonomy coverage check (#693)
feee2aa9 is described below
commit feee2aa9defd64dc90c9ef774d330f0d441215bf
Author: Justin Mclean <ju...@classsoftware.com>
AuthorDate: Fri Jul 3 23:56:44 2026 +1000
feat(validator): add capability taxonomy coverage check (#693)
Reads the Axis 1 (skill) and Axis 2 (tool) capability vocabulary tables
from docs/labels-and-capabilities.md and verifies every taxonomy entry
appears in at least one mapping-table row; entries marked *(reserved)*
or *(future)* are exempt. Cross-checks the SKILL_CAPABILITIES /
TOOL_CAPABILITIES code constants against the parsed vocabulary.
The reserved/future marker accepts an elaborated parenthetical (e.g.
*(future work)*, *(reserved for #999)*), not only the exact forms.
---
.../src/skill_and_tool_validator/__init__.py | 192 +++++++++++++++++
.../tests/test_validator.py | 227 +++++++++++++++++++++
2 files changed, 419 insertions(+)
diff --git a/tools/skill-and-tool-validator/src/skill_and_tool_validator/__init__.py b/tools/skill-and-tool-validator/src/skill_and_tool_validator/__init__.py
index 206efe61..ea4547f5 100644
--- a/tools/skill-and-tool-validator/src/skill_and_tool_validator/__init__.py
+++ b/tools/skill-and-tool-validator/src/skill_and_tool_validator/__init__.py
@@ -111,6 +111,15 @@ skills/:
public branch names must not reveal embargo context. Lines in
explicit "bad example" contexts (containing ``**bad**`` or
``bad:``) are exempt. Advisory only.
+18. Capability taxonomy coverage (SOFT) — reads the Axis 1 (skill) and
+ Axis 2 (tool) capability vocabulary tables from
+ ``docs/labels-and-capabilities.md`` and verifies that every taxonomy
+ entry appears in at least one row of the corresponding mapping table
+ (``## Capability to skill map`` / ``## Capability to tool map``).
+ Vocabulary entries marked ``*(reserved)*`` or ``*(future)*`` in their
+ Definition column are exempted. Also cross-checks that the hardcoded
+ ``SKILL_CAPABILITIES`` and ``TOOL_CAPABILITIES`` code constants match
+ the parsed vocabulary so code and docs stay in sync. Advisory only.
SOFT categories surface as advisory warnings (stderr) without
failing the run unless ``--strict`` is passed.
@@ -472,6 +481,10 @@ TEMPLATE_DRIFT_CATEGORY = "template-drift"
# SOFT advisory: branch name examples in code blocks that contain embargo-breaking
# terms (CVE IDs, security, vulnerability, advisory) before public disclosure.
BRANCH_CONFIDENTIALITY_CATEGORY = "branch-name-confidentiality"
+# SOFT advisory: capability taxonomy vocabulary entry in docs/labels-and-capabilities.md
+# has no skill/tool implementation in the mapping tables, or the hardcoded
+# SKILL_CAPABILITIES / TOOL_CAPABILITIES constants have drifted from the doc.
+CAPABILITY_TAXONOMY_CATEGORY = "capability-taxonomy"
# The `magpie-` namespace prefix every installed framework skill carries.
SKILL_NAME_PREFIX = "magpie-"
@@ -492,6 +505,7 @@ SOFT_CATEGORIES: frozenset[str] = frozenset(
OVERRIDE_CONTRACT_CATEGORY,
TEMPLATE_DRIFT_CATEGORY,
BRANCH_CONFIDENTIALITY_CATEGORY,
+ CAPABILITY_TAXONOMY_CATEGORY,
}
)
HARD_CATEGORIES: frozenset[str] = frozenset(
@@ -2018,6 +2032,181 @@ def validate_capability_sync(root: Path | None = None) -> Iterable[Violation]:
)
+# ---------------------------------------------------------------------------
+# Capability taxonomy coverage check (aspect #18, SOFT)
+# ---------------------------------------------------------------------------
+
+# Section anchors for the Axis 1 and Axis 2 vocabulary tables.
+_AXIS1_ANCHOR = "**Axis 1 — skill capability**"
+_AXIS2_ANCHOR = "**Axis 2 — tool capability**"
+# Marker for reserved or future taxonomy entries in the Definition column.
+# Allows an elaborated parenthetical, e.g. ``*(future work)*`` or
+# ``*(reserved for #999)*`` — only the leading keyword is significant.
+_RESERVED_FUTURE_RE = re.compile(r"\*\((?:reserved|future)\b[^)]*\)\*", re.IGNORECASE)
+
+
+def _parse_capability_vocabulary_tables(
+ text: str,
+) -> tuple[dict[str, bool], dict[str, bool]]:
+ """Parse the Axis 1 and Axis 2 vocabulary tables from ``docs/labels-and-capabilities.md``.
+
+ Returns ``(skill_vocab, tool_vocab)`` where each dict maps a capability
+ token (e.g. ``capability:triage``, ``contract:tracker``) to a bool that
+ is ``True`` when the entry is marked ``*(reserved)*`` or ``*(future)*``
+ in its Definition column and therefore exempt from the coverage check.
+ """
+
+ def _extract_vocab(section_text: str) -> dict[str, bool]:
+ result: dict[str, bool] = {}
+ for line in section_text.splitlines():
+ if not line.startswith("|"):
+ continue
+ if "|---" in line:
+ continue
+ cells = [c.strip() for c in line.strip("|").split("|")]
+ if not cells:
+ continue
+ first = cells[0]
+ # Vocabulary table: first cell contains the capability token itself.
+ m = _CAPABILITY_TOKEN_RE.search(first)
+ if not m:
+ continue
+ token = m.group(1)
+ rest = " ".join(cells[1:])
+ is_exempt = bool(_RESERVED_FUTURE_RE.search(rest))
+ result[token] = is_exempt
+ return result
+
+ axis1_start = text.find(_AXIS1_ANCHOR)
+ axis2_start = text.find(_AXIS2_ANCHOR)
+
+ if axis1_start == -1 or axis2_start == -1:
+ return {}, {}
+
+ axis1_section = text[axis1_start:axis2_start]
+ # Axis 2 ends at the next level-3 heading ("### 3.") or end of document.
+ axis2_end = re.search(r"\n###\s", text[axis2_start:])
+ axis2_section = text[axis2_start : axis2_start + axis2_end.start()] if axis2_end else text[axis2_start:]
+
+ return _extract_vocab(axis1_section), _extract_vocab(axis2_section)
+
+
+def validate_capability_taxonomy_coverage(root: Path | None = None) -> Iterable[Violation]:
+ """Check that every taxonomy vocabulary entry has at least one implementation.
+
+ Reads the Axis 1 (skill) and Axis 2 (tool) vocabulary tables from
+ ``docs/labels-and-capabilities.md``, then verifies that each capability
+ token appears in at least one row of the corresponding mapping table
+ (``## Capability to skill map`` / ``## Capability to tool map``).
+
+ Entries marked ``*(reserved)*`` or ``*(future)*`` in their Definition
+ column are exempted from the coverage requirement — they intentionally
+ have no current implementation.
+
+ Also cross-checks that ``SKILL_CAPABILITIES`` and ``TOOL_CAPABILITIES``
+ code constants match the parsed vocabulary so the two stay in sync.
+
+ All violations are SOFT advisories.
+ """
+ repo_root = root or find_repo_root()
+ doc_path = repo_root / DOCS_LABELS_AND_CAPABILITIES
+ if not doc_path.exists():
+ return
+
+ try:
+ doc_text = doc_path.read_text(encoding="utf-8")
+ except OSError:
+ return
+
+ skill_vocab, tool_vocab = _parse_capability_vocabulary_tables(doc_text)
+ if not skill_vocab and not tool_vocab:
+ yield Violation(
+ doc_path,
+ None,
+ "could not parse Axis 1 / Axis 2 vocabulary tables from "
+ "docs/labels-and-capabilities.md — check that the section anchors are present",
+ category=CAPABILITY_TAXONOMY_CATEGORY,
+ )
+ return
+
+ doc_skills = _parse_capability_doc_table(doc_text, _SKILL_TABLE_HEADER)
+ doc_tools = _parse_capability_doc_table(doc_text, _TOOL_TABLE_HEADER)
+
+ # Collect all capability tokens that appear in the mapping tables.
+ implemented_skill_caps: set[str] = set()
+ for caps in doc_skills.values():
+ implemented_skill_caps |= caps
+ implemented_tool_caps: set[str] = set()
+ for caps in doc_tools.values():
+ implemented_tool_caps |= caps
+
+ # Check Axis 1: every non-exempt vocabulary entry must appear in the skill map.
+ for token, is_exempt in sorted(skill_vocab.items()):
+ if is_exempt:
+ continue
+ if token not in implemented_skill_caps:
+ yield Violation(
+ doc_path,
+ None,
+ f"capability taxonomy entry '{token}' (Axis 1) has no implementation in "
+ f"'## Capability to skill map' — add a skill row or mark the entry "
+ f"*(reserved)* / *(future)* in the vocabulary table",
+ category=CAPABILITY_TAXONOMY_CATEGORY,
+ )
+
+ # Check Axis 2: every non-exempt vocabulary entry must appear in the tool map.
+ for token, is_exempt in sorted(tool_vocab.items()):
+ if is_exempt:
+ continue
+ if token not in implemented_tool_caps:
+ yield Violation(
+ doc_path,
+ None,
+ f"capability taxonomy entry '{token}' (Axis 2) has no implementation in "
+ f"'## Capability to tool map' — add a tool row or mark the entry "
+ f"*(reserved)* / *(future)* in the vocabulary table",
+ category=CAPABILITY_TAXONOMY_CATEGORY,
+ )
+
+ # Cross-check: SKILL_CAPABILITIES code constant vs parsed vocabulary.
+ parsed_skill_set = set(skill_vocab.keys())
+ if parsed_skill_set != SKILL_CAPABILITIES:
+ extra_in_code = SKILL_CAPABILITIES - parsed_skill_set
+ extra_in_doc = parsed_skill_set - SKILL_CAPABILITIES
+ parts = []
+ if extra_in_code:
+ parts.append(f"in code but not in taxonomy: {sorted(extra_in_code)}")
+ if extra_in_doc:
+ parts.append(f"in taxonomy but not in code: {sorted(extra_in_doc)}")
+ yield Violation(
+ doc_path,
+ None,
+ "SKILL_CAPABILITIES constant has drifted from the Axis 1 vocabulary in "
+ f"docs/labels-and-capabilities.md — {'; '.join(parts)}; "
+ "update the constant to match the taxonomy",
+ category=CAPABILITY_TAXONOMY_CATEGORY,
+ )
+
+ # Cross-check: TOOL_CAPABILITIES code constant vs parsed vocabulary.
+ parsed_tool_set = set(tool_vocab.keys())
+ if parsed_tool_set != TOOL_CAPABILITIES:
+ extra_in_code = TOOL_CAPABILITIES - parsed_tool_set
+ extra_in_doc = parsed_tool_set - TOOL_CAPABILITIES
+ parts = []
+ if extra_in_code:
+ parts.append(f"in code but not in taxonomy: {sorted(extra_in_code)}")
+ if extra_in_doc:
+ parts.append(f"in taxonomy but not in code: {sorted(extra_in_doc)}")
+ yield Violation(
+ doc_path,
+ None,
+ "TOOL_CAPABILITIES constant has drifted from the Axis 2 vocabulary in "
+ f"docs/labels-and-capabilities.md — {'; '.join(parts)}; "
+ "update the constant to match the taxonomy",
+ category=CAPABILITY_TAXONOMY_CATEGORY,
+ )
+
+
# ---------------------------------------------------------------------------
# Lowercase -f field check (Pattern 2)
# ---------------------------------------------------------------------------
@@ -3270,6 +3459,9 @@ def run_validation(root: Path | None = None) -> list[Violation]:
continue
violations.extend(validate_branch_name_confidentiality(doc_path, doc_text))
+ # Capability taxonomy coverage: every vocab entry has an implementation or is reserved.
+ violations.extend(validate_capability_taxonomy_coverage(repo_root))
+
return violations
diff --git a/tools/skill-and-tool-validator/tests/test_validator.py b/tools/skill-and-tool-validator/tests/test_validator.py
index 48cdf459..8ad08174 100644
--- a/tools/skill-and-tool-validator/tests/test_validator.py
+++ b/tools/skill-and-tool-validator/tests/test_validator.py
@@ -35,6 +35,7 @@ from skill_and_tool_validator import (
ALLOWED_MODES,
ASF_COUPLING_CATEGORY,
BRANCH_CONFIDENTIALITY_CATEGORY,
+ CAPABILITY_TAXONOMY_CATEGORY,
EVAL_COVERAGE_CATEGORY,
FORBIDDEN_PATTERNS,
GH_LIST_CATEGORY,
@@ -54,11 +55,14 @@ from skill_and_tool_validator import (
PRINCIPLE_CATEGORY,
PRIVACY_CATEGORY,
SECURITY_PATTERN_CATEGORY,
+ SKILL_CAPABILITIES,
SKILL_SOURCE_CATEGORY,
SOFT_CATEGORIES,
STATUS_CATEGORY,
TEMPLATE_DRIFT_CATEGORY,
+ TOOL_CAPABILITIES,
TRIGGER_PRESERVATION_CATEGORY,
+ _parse_capability_vocabulary_tables,
_parse_modes_doc,
_read_mode_table,
collect_doc_files,
@@ -85,6 +89,7 @@ from skill_and_tool_validator import (
validate_asf_coupling,
validate_branch_name_confidentiality,
validate_capability_sync,
+ validate_capability_taxonomy_coverage,
validate_eval_coverage,
validate_frontmatter,
validate_gh_list_limit,
@@ -3209,6 +3214,228 @@ class TestValidateCapabilitySync:
assert violations == [], [v.message for v in violations]
+# ---------------------------------------------------------------------------
+# Capability taxonomy vocabulary parser
+# ---------------------------------------------------------------------------
+
+
+class TestParseCapabilityVocabularyTables:
+ """Tests for _parse_capability_vocabulary_tables."""
+
+ _AXIS1_HEADER = "**Axis 1 — skill capability**"
+ _AXIS2_HEADER = "**Axis 2 — tool capability**"
+
+ def _doc(self, axis1_rows: str = "", axis2_rows: str = "") -> str:
+ return (
+ f"### 2. capability\n\n"
+ f"{self._AXIS1_HEADER} (`capability:*`) — the workflow-lifecycle\n\n"
+ "| Label | Definition |\n"
+ "|---|---|\n"
+ f"{axis1_rows}\n"
+ f"{self._AXIS2_HEADER} (`contract:*` / `substrate:*`) — the interface:\n\n"
+ "| Label | Kind | Definition |\n"
+ "|---|---|---|\n"
+ f"{axis2_rows}\n"
+ "\n### 3. kind:* — change type\n"
+ )
+
+ def test_parses_skill_capabilities(self) -> None:
+ doc = self._doc(
+ axis1_rows=(
+ "| `capability:triage` | Sweep a queue. |\n| `capability:fix` | Implement a fix. |\n"
+ ),
+ )
+ skill_vocab, tool_vocab = _parse_capability_vocabulary_tables(doc)
+ assert "capability:triage" in skill_vocab
+ assert "capability:fix" in skill_vocab
+ assert tool_vocab == {}
+
+ def test_parses_tool_capabilities(self) -> None:
+ doc = self._doc(
+ axis2_rows=(
+ "| `contract:tracker` | contract | Issue backend. |\n"
+ "| `substrate:sandbox` | substrate | Agent isolation. |\n"
+ ),
+ )
+ skill_vocab, tool_vocab = _parse_capability_vocabulary_tables(doc)
+ assert skill_vocab == {}
+ assert "contract:tracker" in tool_vocab
+ assert "substrate:sandbox" in tool_vocab
+
+ def test_reserved_marker_sets_exempt_flag(self) -> None:
+ doc = self._doc(
+ axis1_rows="| `capability:future-thing` | Planned. *(future)* |\n",
+ )
+ skill_vocab, _ = _parse_capability_vocabulary_tables(doc)
+ assert skill_vocab.get("capability:future-thing") is True
+
+ def test_non_exempt_entry_has_false_flag(self) -> None:
+ doc = self._doc(
+ axis1_rows="| `capability:triage` | Sweep. |\n",
+ )
+ skill_vocab, _ = _parse_capability_vocabulary_tables(doc)
+ assert skill_vocab.get("capability:triage") is False
+
+ def test_missing_anchors_returns_empty(self) -> None:
+ skill_vocab, tool_vocab = _parse_capability_vocabulary_tables("No anchors here.")
+ assert skill_vocab == {}
+ assert tool_vocab == {}
+
+
+# ---------------------------------------------------------------------------
+# Capability taxonomy coverage check
+# ---------------------------------------------------------------------------
+
+
+def _seed_taxonomy_repo(
+ tmp_path: Path,
+ *,
+ axis1_rows: str = "",
+ axis2_rows: str = "",
+ mapping_skill_rows: str = "",
+ mapping_tool_rows: str = "",
+) -> Path:
+ """Build a minimal repo with docs/labels-and-capabilities.md for taxonomy tests."""
+ root = tmp_path / "repo"
+ (root / "docs").mkdir(parents=True)
+ (root / "skills").mkdir(parents=True)
+ (root / "tools").mkdir(parents=True)
+
+ doc = (
+ "# Labels and capabilities\n\n"
+ "### 2. capability\n\n"
+ "**Axis 1 — skill capability** (`capability:*`) — lifecycle phase:\n\n"
+ "| Label | Definition |\n"
+ "|---|---|\n"
+ f"{axis1_rows}\n"
+ "**Axis 2 — tool capability** (`contract:*` / `substrate:*`) — interface:\n\n"
+ "| Label | Kind | Definition |\n"
+ "|---|---|---|\n"
+ f"{axis2_rows}\n"
+ "\n### 3. kind:*\n\n"
+ "## Capability to skill map\n\n"
+ "| Skill | Capability / capabilities |\n"
+ "|---|---|\n"
+ f"{mapping_skill_rows}\n"
+ "## Capability to tool map\n\n"
+ "| Tool | Capability / capabilities | Role |\n"
+ "|---|---|---|\n"
+ f"{mapping_tool_rows}\n"
+ )
+ (root / "docs" / "labels-and-capabilities.md").write_text(doc)
+ return root
+
+
+class TestValidateCapabilityTaxonomyCoverage:
+ """Tests for validate_capability_taxonomy_coverage (check #17 — SOFT)."""
+
+ def test_fully_covered_passes(self, tmp_path: Path) -> None:
+ root = _seed_taxonomy_repo(
+ tmp_path,
+ axis1_rows="| `capability:triage` | Sweep. |\n",
+ mapping_skill_rows="| `alpha` | `capability:triage` |\n",
+ )
+ violations = list(validate_capability_taxonomy_coverage(root))
+ taxonomy_violations = [v for v in violations if v.category == CAPABILITY_TAXONOMY_CATEGORY]
+ # The only expected advisory is the SKILL_CAPABILITIES constant drift (test repo
+ # has a one-entry taxonomy while the real constant has ten). Filter to coverage-only.
+ coverage_violations = [v for v in taxonomy_violations if "has no implementation" in v.message]
+ assert coverage_violations == []
+
+ def test_uncovered_axis1_entry_flagged(self, tmp_path: Path) -> None:
+ root = _seed_taxonomy_repo(
+ tmp_path,
+ axis1_rows=("| `capability:triage` | Sweep. |\n| `capability:orphan` | Orphaned. |\n"),
+ mapping_skill_rows="| `alpha` | `capability:triage` |\n",
+ )
+ violations = list(validate_capability_taxonomy_coverage(root))
+ assert any("capability:orphan" in v.message and "Axis 1" in v.message for v in violations)
+
+ def test_uncovered_axis2_entry_flagged(self, tmp_path: Path) -> None:
+ root = _seed_taxonomy_repo(
+ tmp_path,
+ axis2_rows="| `contract:orphan-tool` | contract | Orphaned. |\n",
+ mapping_tool_rows="",
+ )
+ violations = list(validate_capability_taxonomy_coverage(root))
+ assert any("contract:orphan-tool" in v.message and "Axis 2" in v.message for v in violations)
+
+ def test_reserved_entry_is_exempt(self, tmp_path: Path) -> None:
+ root = _seed_taxonomy_repo(
+ tmp_path,
+ axis1_rows="| `capability:future-thing` | Planned. *(reserved)* |\n",
+ mapping_skill_rows="",
+ )
+ violations = list(validate_capability_taxonomy_coverage(root))
+ coverage_violations = [
+ v
+ for v in violations
+ if v.category == CAPABILITY_TAXONOMY_CATEGORY and "has no implementation" in v.message
+ ]
+ assert coverage_violations == []
+
+ def test_future_marker_also_exempt(self, tmp_path: Path) -> None:
+ root = _seed_taxonomy_repo(
+ tmp_path,
+ axis1_rows="| `capability:next-gen` | Next gen. *(future)* |\n",
+ mapping_skill_rows="",
+ )
+ violations = list(validate_capability_taxonomy_coverage(root))
+ coverage_violations = [
+ v
+ for v in violations
+ if v.category == CAPABILITY_TAXONOMY_CATEGORY and "has no implementation" in v.message
+ ]
+ assert coverage_violations == []
+
+ def test_code_constant_drift_flagged(self, tmp_path: Path) -> None:
+ # Axis 1 vocabulary has a token not in SKILL_CAPABILITIES.
+ root = _seed_taxonomy_repo(
+ tmp_path,
+ axis1_rows="| `capability:not-in-code` | New. |\n",
+ mapping_skill_rows="| `alpha` | `capability:not-in-code` |\n",
+ )
+ violations = list(validate_capability_taxonomy_coverage(root))
+ assert any("SKILL_CAPABILITIES constant has drifted" in v.message for v in violations)
+
+ def test_tool_constant_drift_flagged(self, tmp_path: Path) -> None:
+ root = _seed_taxonomy_repo(
+ tmp_path,
+ axis2_rows="| `contract:new-adapter` | contract | New. |\n",
+ mapping_tool_rows="| [`tools/new-adapter`](../tools/new-adapter/) | `contract:new-adapter` | role |\n",
+ )
+ violations = list(validate_capability_taxonomy_coverage(root))
+ assert any("TOOL_CAPABILITIES constant has drifted" in v.message for v in violations)
+
+ def test_category_is_soft(self) -> None:
+ assert CAPABILITY_TAXONOMY_CATEGORY in SOFT_CATEGORIES
+
+ def test_real_repo_passes_clean(self) -> None:
+ """Taxonomy coverage check must pass against the live repo with no violations."""
+ violations = list(validate_capability_taxonomy_coverage())
+ assert violations == [], [v.message for v in violations]
+
+ def test_vocabulary_constants_match_live_taxonomy(self) -> None:
+ """SKILL_CAPABILITIES and TOOL_CAPABILITIES must match the live taxonomy doc."""
+ from pathlib import Path
+
+ doc_path = Path("docs/labels-and-capabilities.md")
+ if not doc_path.exists():
+ pytest.skip("labels-and-capabilities.md not found — not in repo root")
+ doc_text = doc_path.read_text(encoding="utf-8")
+ skill_vocab, tool_vocab = _parse_capability_vocabulary_tables(doc_text)
+ assert set(skill_vocab.keys()) == SKILL_CAPABILITIES, (
+ f"SKILL_CAPABILITIES constant is out of sync with taxonomy. "
+ f"In code only: {SKILL_CAPABILITIES - set(skill_vocab.keys())}; "
+ f"In doc only: {set(skill_vocab.keys()) - SKILL_CAPABILITIES}"
+ )
+ assert set(tool_vocab.keys()) == TOOL_CAPABILITIES, (
+ f"TOOL_CAPABILITIES constant is out of sync with taxonomy. "
+ f"In code only: {TOOL_CAPABILITIES - set(tool_vocab.keys())}; "
+ f"In doc only: {set(tool_vocab.keys()) - TOOL_CAPABILITIES}"
+ )
+
+
# ---------------------------------------------------------------------------
# Eval-coverage check
# ---------------------------------------------------------------------------