You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@magpie.apache.org by po...@apache.org on 2026/10/17 00:21:44 UTC
(magpie) 12/27: fix(generate-cve-json): normalise the body before reading fields and refuse ambiguous duplicates cleanly
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch perf/security-1-import
in repository https://gitbox.apache.org/repos/asf/magpie.git
commit d264adfdc20455648628e1a96b5e4a6187f441d0
Author: Jarek Potiuk <po...@apache.org>
AuthorDate: Fri Oct 16 23:55:05 2026 +0200
fix(generate-cve-json): normalise the body before reading fields and refuse ambiguous duplicates cleanly
Since the switch to github-body-field's parser, a summary heading written
with two spaces after ### was silently missed, leaving the CVE
description empty, and a field repeated with different values ended the
run in a traceback. The body is now normalised in memory against the
template field names first: lenient or differently cased headings name
their field again, CRLF bodies read the same as LF ones, and copies of a
field holding the same value collapse to one. A field repeated with
different values exits 2 with the field name and the heading line
numbers, and no record is written; the tool never picks a copy.
The new --report-json flag writes normalisation_changes, needs_decision
and missing_required (the summary, affected versions, CWE, severity,
credit and CVE link, each marked with whether an agent may propose a
value; never for severity or the CVE link) for the calling skill to act
on. It holds no body text and is written on exit 2 too.
Run over the 379 real tracker bodies against main, every record is the
same except one, which now stops on its differing duplicate; the only
other change is that CRLF bodies no longer carry carriage returns into
the CVE description.
Part of #1672
Generated-by: Claude Opus 5
---
.../cve-tool-vulnogram/generate-cve-json/README.md | 4 +
.../cve-tool-vulnogram/generate-cve-json/SKILL.md | 14 ++
.../src/generate_cve_json/cve_json.py | 173 +++++++++++++++++----
.../tests/test_normalised_body.py | 96 ++++++++++++
4 files changed, 259 insertions(+), 28 deletions(-)
diff --git a/tools/cve-tool-vulnogram/generate-cve-json/README.md b/tools/cve-tool-vulnogram/generate-cve-json/README.md
index 2f8c5dd5..4e23445a 100644
--- a/tools/cve-tool-vulnogram/generate-cve-json/README.md
+++ b/tools/cve-tool-vulnogram/generate-cve-json/README.md
@@ -74,6 +74,10 @@ uv run generate-cve-json <ISSUE-NUMBER>
Flags are documented in `generate-cve-json --help` and in [`SKILL.md`](SKILL.md).
+The body is normalised in memory with [`github-body-field`](../../github-body-field/README.md)'s `normalise` before any field is read.
+A field repeated with different values exits 2 with the field name and heading line numbers, never a guess.
+`--report-json <path>` writes `normalisation_changes`, `needs_decision` and `missing_required` (no body text) for the calling skill to act on.
+
## Record a published CVE in the project's security records — `advisory-records`
A second console script in the same project.
diff --git a/tools/cve-tool-vulnogram/generate-cve-json/SKILL.md b/tools/cve-tool-vulnogram/generate-cve-json/SKILL.md
index f2b1e052..3a117387 100644
--- a/tools/cve-tool-vulnogram/generate-cve-json/SKILL.md
+++ b/tools/cve-tool-vulnogram/generate-cve-json/SKILL.md
@@ -456,6 +456,20 @@ Additional flags, all optional:
tab is in "inner block only" mode.
- `--stdin` — read the issue body from stdin instead of calling `gh`.
Useful for offline iteration and for drafting by hand.
+- `--report-json <path>` — also write a machine-readable report (field names, line numbers and kinds; never body text):
+ `normalisation_changes` (what the in-memory normalisation below rewrote),
+ `needs_decision` (fields repeated with different values),
+ and `missing_required` (empty required fields, each with `agent_may_propose`; `false` for *Severity* and *CVE tool link*).
+ It is written before the record is built, so it exists on exit 2 too.
+ The calling skill acts on it; the tool itself never calls a model.
+
+Before reading any field the script normalises the body **in memory** with
+[`github-body-field`](../../github-body-field/README.md)'s `normalise`:
+CRLF becomes LF, a heading written `### Name`, `###<tab>Name`, indented, with closing `#`s or in another case still names its template field,
+a heading-shaped line inside a code fence never counts,
+and copies of a field that hold the same value collapse to one.
+A field repeated outside fences with **different** values is never guessed:
+the script exits 2 with an error naming the field and the heading line numbers, and writes no record.
The script is deterministic — re-running it with the same flags and the
same tracking-issue body produces the same JSON bytes.
diff --git a/tools/cve-tool-vulnogram/generate-cve-json/src/generate_cve_json/cve_json.py b/tools/cve-tool-vulnogram/generate-cve-json/src/generate_cve_json/cve_json.py
index f410de69..ecc7939a 100644
--- a/tools/cve-tool-vulnogram/generate-cve-json/src/generate_cve_json/cve_json.py
+++ b/tools/cve-tool-vulnogram/generate-cve-json/src/generate_cve_json/cve_json.py
@@ -447,7 +447,85 @@ NO_RESPONSE = "_No response_"
class DuplicateFieldError(ValueError):
- """A `### <field>` heading appears more than once outside code fences."""
+ """A `### <field>` heading appears more than once outside code fences, with different values."""
+
+
+# The template fields this tool reads, in body order. They are the known
+# field names `normalise_body` canonicalises headings against.
+TEMPLATE_FIELDS: tuple[str, ...] = (
+ "Short public summary for publish",
+ "Affected versions",
+ "Security mailing list thread",
+ "Public advisory URL",
+ "Reporter credited as",
+ "PR with the fix",
+ "Remediation developer",
+ "CWE",
+ "Severity",
+ "CVE tool link",
+)
+
+# The fields a CVE record cannot go to review without: the description, the
+# affected range, the weakness, the severity, the credit and the CVE ID
+# (see `_is_cna_ready_for_review`). The fix-PR reference is not here: it is
+# legitimately empty until the fix exists.
+REQUIRED_FIELDS: tuple[str, ...] = (
+ "Short public summary for publish",
+ "Affected versions",
+ "CWE",
+ "Severity",
+ "Reporter credited as",
+ "CVE tool link",
+)
+
+# Required fields an agent must never propose from the report text: the
+# severity is the team's independent score (AGENTS.md: a reporter's CVSS is
+# informational only), and the CVE ID comes from the allocation step.
+TEAM_ONLY_FIELDS: frozenset[str] = frozenset({"Severity", "CVE tool link"})
+
+# Severity values that mean "not scored yet".
+_UNSCORED_SEVERITY = {"", "unknown", NO_RESPONSE.lower()}
+
+
+def normalise_body(body: str) -> body_field_parser.NormaliseResult:
+ """``body`` normalised by github-body-field against :data:`TEMPLATE_FIELDS`.
+
+ In memory only: this tool never rewrites the tracker body's fields.
+ """
+ return body_field_parser.normalise(body, TEMPLATE_FIELDS)
+
+
+def _duplicate_message(decision: dict) -> str:
+ lines = ", ".join(str(n) for n in decision.get("lines") or [])
+ return (
+ f"tracker body has more than one `### {decision['field']}` section outside code fences "
+ f"with different values (lines {lines}); refusing to choose one — fix the body first"
+ )
+
+
+def _value_of(normalised: body_field_parser.NormaliseResult, heading: str) -> str:
+ for decision in normalised.needs_decision:
+ if decision["field"] == heading:
+ raise DuplicateFieldError(_duplicate_message(decision))
+ try:
+ value = body_field_parser.extract_field(normalised.text, heading).strip()
+ except body_field_parser.FieldNotFoundError:
+ return ""
+ return "" if value == NO_RESPONSE else value
+
+
+def missing_required(values: dict[str, str]) -> list[dict]:
+ """The :data:`REQUIRED_FIELDS` that ``values`` leaves empty, each marked
+ with whether an agent may propose a value for it."""
+ missing = []
+ for name in REQUIRED_FIELDS:
+ if name not in values:
+ continue # undecidable (a duplicate): reported under needs_decision
+ value = values[name].strip()
+ empty = value.lower() in _UNSCORED_SEVERITY if name == "Severity" else not value
+ if empty:
+ missing.append({"field": name, "agent_may_propose": name not in TEAM_ONLY_FIELDS})
+ return missing
# Sentinel token for an as-yet-unreleased upper bound in
@@ -544,29 +622,22 @@ def fetch_issue(issue_number: int | str, repo: str) -> tuple[str, str, list[str]
def extract_field(body: str, heading: str) -> str:
- """Return the text under ``### <heading>`` up to the next ``###``.
+ """Return the text under ``### <heading>`` up to the next heading.
- The section is found by github-body-field's fence-aware parser, so a
- ``### <heading>`` line inside a fenced code block in the report is never
- taken for the field, and a heading that appears twice outside fences
- raises :class:`DuplicateFieldError` instead of the first one winning.
+ The body is first normalised in memory (:func:`normalise_body`): CRLF
+ becomes LF, a heading written ``### Name``, ``###<tab>Name`` or in another
+ case still names its field, and copies of a field holding the same value
+ collapse to one. The section is then found by github-body-field's
+ CommonMark-fence-aware parser, so a ``### <heading>`` line inside a
+ fenced code block in the report is never taken for the field, and a
+ heading that appears twice outside fences with different values raises
+ :class:`DuplicateFieldError` instead of the first one winning.
Trailing whitespace is stripped. The literal placeholder
``_No response_`` is normalised to an empty string so callers only
have to check truthiness.
"""
- try:
- value = body_field_parser.extract_field(body, heading).strip()
- except body_field_parser.FieldNotFoundError as exc:
- if "appears" in str(exc):
- raise DuplicateFieldError(
- f"tracker body has more than one `### {heading}` section outside code fences; "
- "refusing to choose one — fix the body first"
- ) from exc
- return ""
- if value == NO_RESPONSE:
- return ""
- return value
+ return _value_of(normalise_body(body), heading)
def parse_credits_from_field(value: str) -> list[str]:
@@ -2171,6 +2242,17 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
type=Path,
help=("Write the resulting JSON to this path. When omitted the JSON is printed to stdout."),
)
+ parser.add_argument(
+ "--report-json",
+ type=Path,
+ help=(
+ "Also write a machine-readable report to this path: the in-memory "
+ "body normalisation's `normalisation_changes`, the `needs_decision` "
+ "duplicates that stop the run (exit 2), and the `missing_required` "
+ "fields, each with `agent_may_propose`. Written before the record "
+ "is built, so it exists on exit 2 too. Never holds body text."
+ ),
+ )
parser.add_argument(
"--stdin",
action="store_true",
@@ -2356,6 +2438,24 @@ def resolve_title(
return title
+def write_report(
+ path: Path,
+ normalised: body_field_parser.NormaliseResult,
+ blocking: list[dict],
+ missing: list[dict],
+) -> None:
+ """Write the ``--report-json`` file: what the in-memory normalisation
+ changed, what it left for a human to decide, and which required fields
+ are empty. Field names, line numbers and kinds only, never body text."""
+ report = {
+ "normalisation_changes": normalised.changes,
+ "needs_decision": blocking,
+ "missing_required": missing,
+ }
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")
+
+
def emit_json(obj: dict, output: Path | None) -> str:
text = json.dumps(obj, indent=4, sort_keys=True, ensure_ascii=False)
if output:
@@ -2460,16 +2560,33 @@ def main(argv: list[str] | None = None) -> int:
else:
release_vote_in_progress = None # legacy: ready ⇒ REVIEW
- summary = extract_field(body, "Short public summary for publish")
- affected_field = extract_field(body, "Affected versions")
- mailing_list = extract_field(body, "Security mailing list thread")
- public_advisory_field = extract_field(body, "Public advisory URL")
- credited_as = extract_field(body, "Reporter credited as")
- pr_field = extract_field(body, "PR with the fix")
- remediation_developer_field = extract_field(body, "Remediation developer")
- cwe = extract_field(body, "CWE")
- severity = extract_field(body, "Severity")
- cve_tool = extract_field(body, "CVE tool link")
+ # Normalise once, in memory, then read every field from the canonical body.
+ # A field repeated with different values is a decision for a human: stop
+ # cleanly (exit 2) and name it, never pick a copy.
+ normalised = normalise_body(body)
+ blocking = [d for d in normalised.needs_decision if d["field"] in TEMPLATE_FIELDS]
+ blocked = {str(d["field"]) for d in blocking}
+ values = {name: _value_of(normalised, name) for name in TEMPLATE_FIELDS if name not in blocked}
+ if args.report_json is not None:
+ missing = missing_required(values)
+ if args.cve_id:
+ missing = [m for m in missing if m["field"] != "CVE tool link"]
+ write_report(args.report_json, normalised, blocking, missing)
+ if blocking:
+ for decision in blocking:
+ print(f"error: {_duplicate_message(decision)}", file=sys.stderr)
+ return 2
+
+ summary = values["Short public summary for publish"]
+ affected_field = values["Affected versions"]
+ mailing_list = values["Security mailing list thread"]
+ public_advisory_field = values["Public advisory URL"]
+ credited_as = values["Reporter credited as"]
+ pr_field = values["PR with the fix"]
+ remediation_developer_field = values["Remediation developer"]
+ cwe = values["CWE"]
+ severity = values["Severity"]
+ cve_tool = values["CVE tool link"]
cve_id = args.cve_id or parse_cve_id(cve_tool)
title = resolve_title(issue_title, summary, args.title)
diff --git a/tools/cve-tool-vulnogram/generate-cve-json/tests/test_normalised_body.py b/tools/cve-tool-vulnogram/generate-cve-json/tests/test_normalised_body.py
new file mode 100644
index 00000000..9de84236
--- /dev/null
+++ b/tools/cve-tool-vulnogram/generate-cve-json/tests/test_normalised_body.py
@@ -0,0 +1,96 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""The in-memory body normalisation, the duplicate refusal and `--report-json`."""
+
+from __future__ import annotations
+
+import io
+import json
+
+from generate_cve_json import cve_json
+
+from .test_cli import _issue_body
+
+
+def _heading_lines(body: str, name: str) -> list[int]:
+ return [n for n, line in enumerate(body.split("\n"), 1) if line == f"### {name}"]
+
+
+def test_lenient_heading_and_crlf_still_carry_the_summary(monkeypatch, capsys):
+ summary = "A long enough public summary of the flaw."
+ body = _issue_body(summary=summary).replace(
+ "### Short public summary for publish", "### Short public summary for publish"
+ )
+ monkeypatch.setattr("sys.stdin", io.StringIO(body.replace("\n", "\r\n")))
+ assert cve_json.main(["--stdin", "--no-envelope"]) == 0
+ record = json.loads(capsys.readouterr().out)
+ assert record["descriptions"][0]["value"] == summary
+
+
+def test_heading_case_is_canonicalised():
+ body = _issue_body().replace("### CWE", "### cwe")
+ assert cve_json.extract_field(body, "CWE") == "CWE-352"
+
+
+def test_equal_duplicates_collapse():
+ body = _issue_body() + "\n### CWE\nCWE-352\n"
+ assert cve_json.extract_field(body, "CWE") == "CWE-352"
+
+
+def test_differing_duplicates_exit_2_cleanly_and_report(monkeypatch, tmp_path, capsys):
+ body = _issue_body() + "\n### Affected versions\n< 9.9.9\n"
+ lines = _heading_lines(body, "Affected versions")
+ assert len(lines) == 2
+ monkeypatch.setattr("sys.stdin", io.StringIO(body))
+ report = tmp_path / "report.json"
+ assert cve_json.main(["--stdin", "--report-json", str(report)]) == 2
+ captured = capsys.readouterr()
+ assert captured.out == ""
+ assert "Traceback" not in captured.err
+ assert "`### Affected versions`" in captured.err
+ assert f"lines {lines[0]}, {lines[1]}" in captured.err
+ data = json.loads(report.read_text())
+ assert data["needs_decision"] == [{"field": "Affected versions", "kind": "duplicate", "lines": lines}]
+ assert data["missing_required"] == []
+ assert "9.9.9" not in report.read_text() # the report never carries body text
+
+
+def test_report_lists_missing_required_fields(monkeypatch, tmp_path):
+ body = _issue_body(cwe="_No response_", severity="Unknown", cve_tool="", credited="")
+ body = body.replace("### Severity", "### Severity")
+ monkeypatch.setattr("sys.stdin", io.StringIO(body))
+ report = tmp_path / "report.json"
+ assert cve_json.main(["--stdin", "--cve-id", "CVE-2026-1", "--report-json", str(report)]) == 0
+ data = json.loads(report.read_text())
+ assert data["needs_decision"] == []
+ assert data["missing_required"] == [
+ {"field": "CWE", "agent_may_propose": True},
+ {"field": "Severity", "agent_may_propose": False},
+ {"field": "Reporter credited as", "agent_may_propose": True},
+ ]
+ assert [c["kind"] for c in data["normalisation_changes"]] == ["heading"]
+
+
+def test_missing_cve_link_is_team_only():
+ assert cve_json.missing_required({"CVE tool link": ""}) == [
+ {"field": "CVE tool link", "agent_may_propose": False}
+ ]
+
+
+def test_four_backtick_block_does_not_hide_later_fields():
+ body = "### The issue description\n\n````\n```\n### CWE\nCWE-1\n```\n````\n\n" + _issue_body()
+ assert cve_json.extract_field(body, "CWE") == "CWE-352"