You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@magpie.apache.org by po...@apache.org on 2026/10/05 22:31:16 UTC
(magpie) branch main updated: test(release-verify-rc): grade Step 6b paste recipes by their rules, not one reference text (#1536)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/magpie.git
The following commit(s) were added to refs/heads/main by this push:
new 6c1f38ca test(release-verify-rc): grade Step 6b paste recipes by their rules, not one reference text (#1536)
6c1f38ca is described below
commit 6c1f38cad758b7f515a2b611087e40efc3d06875
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Tue Oct 6 00:31:09 2026 +0200
test(release-verify-rc): grade Step 6b paste recipes by their rules, not one reference text (#1536)
The Step 6b suite flaked 1-2 of 6 cases on `paste_recipe`, on `main` too:
the grader compared each candidate recipe with one reference recipe word
for word, so correct answers failed for quoting the `gpg --verify`
arguments differently or for adding the companion verification lines.
Replace the exact `paste_recipe` in every case with structural checks in
a new `assertions.json`, encoding the output-spec rule: the recipe invokes
`maven-artifact-verify` on the staged directory, sets `--digests sha512`
(the digest set every case configures), and passes `--podling` only when
the source artefact ships a `DISCLAIMER`. Every original reference recipe
satisfies them. Same treatment as the Step 6c suite in #1505.
Generated-by: Claude Opus 5
---
.../step-6b-jvm-artefacts/fixtures/assertions.json | 17 +++++++++++++++++
.../case-1-clean-podling-staged-set/expected.json | 4 +++-
.../fixtures/case-2-wrong-pom-licence/expected.json | 4 +++-
.../case-3-missing-companion-signature/expected.json | 4 +++-
.../fixtures/case-4-jar-absent-nexus-only/expected.json | 12 +++++++++---
.../case-5-observations-never-fail/expected.json | 4 +++-
.../case-6-namespace-outside-org-apache/expected.json | 4 +++-
.../step-6b-jvm-artefacts/fixtures/output-spec.md | 5 ++++-
8 files changed, 45 insertions(+), 9 deletions(-)
diff --git a/tools/skill-evals/evals/release-verify-rc/step-6b-jvm-artefacts/fixtures/assertions.json b/tools/skill-evals/evals/release-verify-rc/step-6b-jvm-artefacts/fixtures/assertions.json
new file mode 100644
index 00000000..a8e927dc
--- /dev/null
+++ b/tools/skill-evals/evals/release-verify-rc/step-6b-jvm-artefacts/fixtures/assertions.json
@@ -0,0 +1,17 @@
+{
+ "has_tool_invocation": {
+ "type": "regex",
+ "field": "paste_recipe",
+ "pattern": "maven-artifact-verify[^\\n]*\\s[\"']?dist/dev/[^\\s\"']+-rc[0-9]+"
+ },
+ "has_sha512_digests": {
+ "type": "regex",
+ "field": "paste_recipe",
+ "pattern": "--digests[=\\s]+[\"']?sha512"
+ },
+ "has_podling_flag": {
+ "type": "regex",
+ "field": "paste_recipe",
+ "pattern": "maven-artifact-verify[^\\n]*\\s--podling\\b"
+ }
+}
diff --git a/tools/skill-evals/evals/release-verify-rc/step-6b-jvm-artefacts/fixtures/case-1-clean-podling-staged-set/expected.json b/tools/skill-evals/evals/release-verify-rc/step-6b-jvm-artefacts/fixtures/case-1-clean-podling-staged-set/expected.json
index f10ac6a8..4c0ec8d3 100644
--- a/tools/skill-evals/evals/release-verify-rc/step-6b-jvm-artefacts/fixtures/case-1-clean-podling-staged-set/expected.json
+++ b/tools/skill-evals/evals/release-verify-rc/step-6b-jvm-artefacts/fixtures/case-1-clean-podling-staged-set/expected.json
@@ -3,5 +3,7 @@
"status": "PASS",
"pom_findings": [],
"companion_findings": [],
- "paste_recipe": "uv run --project .apache-magpie/tools/maven-artifact-verify maven-artifact-verify \"dist/dev/foo/1.0.0-rc1\" --digests sha512 --podling\ngpg --verify \"dist/dev/foo/1.0.0-rc1/foo-core-1.0.0-sources.jar.asc\" \"dist/dev/foo/1.0.0-rc1/foo-core-1.0.0-sources.jar\"\ngpg --verify \"dist/dev/foo/1.0.0-rc1/foo-core-1.0.0-javadoc.jar.asc\" \"dist/dev/foo/1.0.0-rc1/foo-core-1.0.0-javadoc.jar\""
+ "has_tool_invocation": true,
+ "has_sha512_digests": true,
+ "has_podling_flag": true
}
diff --git a/tools/skill-evals/evals/release-verify-rc/step-6b-jvm-artefacts/fixtures/case-2-wrong-pom-licence/expected.json b/tools/skill-evals/evals/release-verify-rc/step-6b-jvm-artefacts/fixtures/case-2-wrong-pom-licence/expected.json
index 6bf81d63..30b15927 100644
--- a/tools/skill-evals/evals/release-verify-rc/step-6b-jvm-artefacts/fixtures/case-2-wrong-pom-licence/expected.json
+++ b/tools/skill-evals/evals/release-verify-rc/step-6b-jvm-artefacts/fixtures/case-2-wrong-pom-licence/expected.json
@@ -5,5 +5,7 @@
"foo-core-1.0.0.pom: licences declared but none is ALv2 (expected name matching 'Apache License, Version 2.0' or url containing apache.org/licenses/LICENSE-2.0)"
],
"companion_findings": [],
- "paste_recipe": "uv run --project .apache-magpie/tools/maven-artifact-verify maven-artifact-verify \"dist/dev/foo/1.0.0-rc1\" --digests sha512 --podling\ngpg --verify \"dist/dev/foo/1.0.0-rc1/foo-core-1.0.0-sources.jar.asc\" \"dist/dev/foo/1.0.0-rc1/foo-core-1.0.0-sources.jar\"\ngpg --verify \"dist/dev/foo/1.0.0-rc1/foo-core-1.0.0-javadoc.jar.asc\" \"dist/dev/foo/1.0.0-rc1/foo-core-1.0.0-javadoc.jar\""
+ "has_tool_invocation": true,
+ "has_sha512_digests": true,
+ "has_podling_flag": true
}
diff --git a/tools/skill-evals/evals/release-verify-rc/step-6b-jvm-artefacts/fixtures/case-3-missing-companion-signature/expected.json b/tools/skill-evals/evals/release-verify-rc/step-6b-jvm-artefacts/fixtures/case-3-missing-companion-signature/expected.json
index 078886ce..23795625 100644
--- a/tools/skill-evals/evals/release-verify-rc/step-6b-jvm-artefacts/fixtures/case-3-missing-companion-signature/expected.json
+++ b/tools/skill-evals/evals/release-verify-rc/step-6b-jvm-artefacts/fixtures/case-3-missing-companion-signature/expected.json
@@ -5,5 +5,7 @@
"companion_findings": [
"foo-core-1.0.0-sources.jar: missing .asc signature file (foo-core-1.0.0-sources.jar.asc)"
],
- "paste_recipe": "uv run --project .apache-magpie/tools/maven-artifact-verify maven-artifact-verify \"dist/dev/foo/1.0.0-rc1\" --digests sha512\ngpg --verify \"dist/dev/foo/1.0.0-rc1/foo-core-1.0.0-javadoc.jar.asc\" \"dist/dev/foo/1.0.0-rc1/foo-core-1.0.0-javadoc.jar\""
+ "has_tool_invocation": true,
+ "has_sha512_digests": true,
+ "has_podling_flag": false
}
diff --git a/tools/skill-evals/evals/release-verify-rc/step-6b-jvm-artefacts/fixtures/case-4-jar-absent-nexus-only/expected.json b/tools/skill-evals/evals/release-verify-rc/step-6b-jvm-artefacts/fixtures/case-4-jar-absent-nexus-only/expected.json
index fd23a551..c39f7fd2 100644
--- a/tools/skill-evals/evals/release-verify-rc/step-6b-jvm-artefacts/fixtures/case-4-jar-absent-nexus-only/expected.json
+++ b/tools/skill-evals/evals/release-verify-rc/step-6b-jvm-artefacts/fixtures/case-4-jar-absent-nexus-only/expected.json
@@ -1,7 +1,13 @@
{
"step": "jvm-artefacts",
"status": "WARN",
- "pom_findings": ["foo-core-1.0.0.pom: check1 developers INHERITED-UNVERIFIED — element absent and the parent chain is not fully staged locally; verify against the effective POM"],
- "companion_findings": ["foo-core-1.0.0.jar is declared by a staged POM but not staged locally (ABSENT); jvm_companion_location: nexus-staging — an observation, not a failure"],
- "paste_recipe": "uv run --project .apache-magpie/tools/maven-artifact-verify maven-artifact-verify \"dist/dev/foo/1.0.0-rc1\" --digests sha512"
+ "pom_findings": [
+ "foo-core-1.0.0.pom: check1 developers INHERITED-UNVERIFIED — element absent and the parent chain is not fully staged locally; verify against the effective POM"
+ ],
+ "companion_findings": [
+ "foo-core-1.0.0.jar is declared by a staged POM but not staged locally (ABSENT); jvm_companion_location: nexus-staging — an observation, not a failure"
+ ],
+ "has_tool_invocation": true,
+ "has_sha512_digests": true,
+ "has_podling_flag": false
}
diff --git a/tools/skill-evals/evals/release-verify-rc/step-6b-jvm-artefacts/fixtures/case-5-observations-never-fail/expected.json b/tools/skill-evals/evals/release-verify-rc/step-6b-jvm-artefacts/fixtures/case-5-observations-never-fail/expected.json
index 7c281225..3d85716b 100644
--- a/tools/skill-evals/evals/release-verify-rc/step-6b-jvm-artefacts/fixtures/case-5-observations-never-fail/expected.json
+++ b/tools/skill-evals/evals/release-verify-rc/step-6b-jvm-artefacts/fixtures/case-5-observations-never-fail/expected.json
@@ -9,5 +9,7 @@
"foo-core-1.0.0-sources.jar: contains 3 .class entries (compiled code in the sources companion) - observation only, never a failure",
"foo-core-1.0.0-javadoc.jar: placeholder (empty or MANIFEST-only) - a Maven-Central-sanctioned pattern, not a defect"
],
- "paste_recipe": "uv run --project .apache-magpie/tools/maven-artifact-verify maven-artifact-verify \"dist/dev/foo/1.0.0-rc1\" --digests sha512 --podling"
+ "has_tool_invocation": true,
+ "has_sha512_digests": true,
+ "has_podling_flag": true
}
diff --git a/tools/skill-evals/evals/release-verify-rc/step-6b-jvm-artefacts/fixtures/case-6-namespace-outside-org-apache/expected.json b/tools/skill-evals/evals/release-verify-rc/step-6b-jvm-artefacts/fixtures/case-6-namespace-outside-org-apache/expected.json
index 01af941a..dde4849c 100644
--- a/tools/skill-evals/evals/release-verify-rc/step-6b-jvm-artefacts/fixtures/case-6-namespace-outside-org-apache/expected.json
+++ b/tools/skill-evals/evals/release-verify-rc/step-6b-jvm-artefacts/fixtures/case-6-namespace-outside-org-apache/expected.json
@@ -9,5 +9,7 @@
"foo-core-2.1.0-sources.jar: 31 .java/.scala/.kt source entries; no .class entries",
"foo-core-2.1.0-javadoc.jar: 58 non-META-INF entries; documentation layout is not judged"
],
- "paste_recipe": "uv run --project .apache-magpie/tools/maven-artifact-verify maven-artifact-verify \"dist/dev/foo/2.1.0-rc1\" --digests sha512"
+ "has_tool_invocation": true,
+ "has_sha512_digests": true,
+ "has_podling_flag": false
}
diff --git a/tools/skill-evals/evals/release-verify-rc/step-6b-jvm-artefacts/fixtures/output-spec.md b/tools/skill-evals/evals/release-verify-rc/step-6b-jvm-artefacts/fixtures/output-spec.md
index c99cfdab..8b26db19 100644
--- a/tools/skill-evals/evals/release-verify-rc/step-6b-jvm-artefacts/fixtures/output-spec.md
+++ b/tools/skill-evals/evals/release-verify-rc/step-6b-jvm-artefacts/fixtures/output-spec.md
@@ -48,5 +48,8 @@ Grading rules:
set from `jvm_digest_set` when `release-build.md § JVM artefact
checks` sets it, otherwise from the § Digest set, and `--podling`
passed only when the source artefact ships a `DISCLAIMER` /
- `DISCLAIMER-WIP`.
+ `DISCLAIMER-WIP`. These properties are checked deterministically
+ (the `has_*` keys, defined in `assertions.json`) rather than
+ against one reference recipe; extra lines such as `gpg --verify`
+ for the companions are fine.
- No extra keys are permitted in the response.