You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@magpie.apache.org by po...@apache.org on 2026/10/05 15:09:05 UTC
(magpie) branch main updated: fix(skill-evals): give template-less eval steps a neutral user prompt (#1524)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/magpie.git
The following commit(s) were added to refs/heads/main by this push:
new 215067f8 fix(skill-evals): give template-less eval steps a neutral user prompt (#1524)
215067f8 is described below
commit 215067f8d46e0c9c12fcc97a9f867bf100554717
Author: Davide Polato <dp...@apache.org>
AuthorDate: Mon Oct 5 17:08:58 2026 +0200
fix(skill-evals): give template-less eval steps a neutral user prompt (#1524)
The runner's default user prompt, used by every step without a
user-prompt-template.md, was the security-issue-import Step 2a template.
It framed each case as an incoming report checked against a tracker
corpus and a reporter roster, and asked the model to "apply the semantic
sweep and reporter-identity check". 33 other steps (the release-* steps,
reviewer-routing and non-asf-profile-smoke) received that framing, with
an empty corpus and a "(none)" roster, next to a system prompt for an
unrelated task.
The default is now the case report followed by "Return JSON only.".
security-issue-import/step-2a-semantic-sweep, the step the old default
was written for, gets its own user-prompt-template.md with the old text,
so its rendered prompt is unchanged apart from the SPDX comment that
every template file carries.
Closes #1492
Signed-off-by: Davide Polato <dp...@apache.org>
---
.../fixtures/user-prompt-template.md | 16 ++++
tools/skill-evals/src/skill_evals/runner.py | 15 +---
tools/skill-evals/tests/test_runner.py | 88 +++++++++++++++++++---
3 files changed, 96 insertions(+), 23 deletions(-)
diff --git a/tools/skill-evals/evals/security-issue-import/step-2a-semantic-sweep/fixtures/user-prompt-template.md b/tools/skill-evals/evals/security-issue-import/step-2a-semantic-sweep/fixtures/user-prompt-template.md
new file mode 100644
index 00000000..db710e15
--- /dev/null
+++ b/tools/skill-evals/evals/security-issue-import/step-2a-semantic-sweep/fixtures/user-prompt-template.md
@@ -0,0 +1,16 @@
+<!-- SPDX-License-Identifier: Apache-2.0
+ https://www.apache.org/licenses/LICENSE-2.0 -->
+
+## Existing open trackers (corpus)
+
+{corpus}
+
+## Reporter roster (existing trackers mapped to reporter email)
+
+{roster}
+
+## Incoming report
+
+{report}
+
+Apply the semantic sweep and reporter-identity check. Return JSON only.
diff --git a/tools/skill-evals/src/skill_evals/runner.py b/tools/skill-evals/src/skill_evals/runner.py
index fbd786df..f1d27071 100644
--- a/tools/skill-evals/src/skill_evals/runner.py
+++ b/tools/skill-evals/src/skill_evals/runner.py
@@ -90,23 +90,14 @@ from pathlib import Path
# Prompt construction
# ---------------------------------------------------------------------------
-# Available slots: {corpus}, {roster}, {report}.
+# Used when a fixtures dir has no user-prompt-template.md.
+# Slots a custom user-prompt-template.md may use: {corpus}, {roster}, {report}.
# Literal braces in a custom user-prompt-template.md that are NOT slots
# must be doubled ({{ and }}) so Python's str.format() leaves them intact.
USER_PROMPT_TEMPLATE = """\
-## Existing open trackers (corpus)
-
-{corpus}
-
-## Reporter roster (existing trackers mapped to reporter email)
-
-{roster}
-
-## Incoming report
-
{report}
-Apply the semantic sweep and reporter-identity check. Return JSON only.
+Return JSON only.
"""
diff --git a/tools/skill-evals/tests/test_runner.py b/tools/skill-evals/tests/test_runner.py
index 05d4557c..e0ca7b6c 100644
--- a/tools/skill-evals/tests/test_runner.py
+++ b/tools/skill-evals/tests/test_runner.py
@@ -372,17 +372,6 @@ def test_load_step_config_uses_custom_user_prompt_template(tmp_path: Path):
assert user_prompt_template == "Custom: {report}"
-def test_load_step_config_uses_default_user_prompt_template_when_absent(tmp_path: Path):
- fixtures_dir = _make_fixtures_dir(
- tmp_path / "step-dir",
- system_prompt="System.",
- )
- _, user_prompt_template = load_step_config(fixtures_dir)
- assert "{corpus}" in user_prompt_template
- assert "{roster}" in user_prompt_template
- assert "{report}" in user_prompt_template
-
-
def test_load_step_config_raises_when_neither_config_present(tmp_path: Path):
fixtures_dir = tmp_path / "empty-fixtures"
fixtures_dir.mkdir()
@@ -732,6 +721,83 @@ def test_main_bad_user_prompt_template_raises(tmp_path: Path):
main([str(fixtures_dir)])
+def _printed_user_prompt(capsys: pytest.CaptureFixture[str], case_dir: Path) -> str:
+ """Return the user prompt ``main`` prints for ``case_dir`` in print mode."""
+ rc, stdout, _ = _run_main(capsys, [str(case_dir)])
+ assert rc == 0
+ return stdout.split("--- USER PROMPT ---\n", 1)[1].split("\n--- EXPECTED ---\n", 1)[0]
+
+
+def test_main_default_user_prompt_holds_the_report_without_import_framing(
+ tmp_path: Path, capsys: pytest.CaptureFixture[str]
+):
+ """A step without user-prompt-template.md gets a neutral user turn, not the import step's."""
+ fixtures_dir = _make_fixtures_dir(tmp_path / "step-dir", system_prompt="System.")
+ case_dir = _make_case(fixtures_dir, "case-1", report="The incoming report.")
+
+ user_prompt = _printed_user_prompt(capsys, case_dir)
+
+ assert "The incoming report." in user_prompt
+ for import_text in ("Existing open trackers", "Reporter roster", "semantic sweep"):
+ assert import_text not in user_prompt
+
+
+_IMPORT_SWEEP_CASE = (
+ Path(__file__).parents[1]
+ / "evals/security-issue-import/step-2a-semantic-sweep/fixtures/case-1-clear-duplicate"
+)
+
+_SPDX_HEADER = (
+ "<!-- SPDX-License-Identifier: Apache-2.0\n https://www.apache.org/licenses/LICENSE-2.0 -->"
+)
+
+# User prompt for semantic-sweep case-1, without the template's SPDX header.
+_IMPORT_SWEEP_CASE_1_USER_PROMPT = """\
+## Existing open trackers (corpus)
+
+#101 | 'Webserver: unauthenticated access to DAG run history via REST API'
+Body: An unauthenticated remote attacker can query /api/v1/dags/{dag_id}/dagRuns and retrieve full execution history including task logs without any credentials. Tested on Airflow 2.9.1. The endpoint lacks an auth check in airflow/api/
+
+#102 | 'Providers/SFTP: path traversal in SFTPHook when handling remote paths'
+Body: SFTPHook.retrieve_file() does not sanitise the remote_path argument. An operator-configured DAG can supply ../../../etc/passwd as remote_path and read arbitrary files from the SFTP server's host. Affected: airflow/providers/sftp/hooks/sftp.py
+
+#103 | 'API: SSRF via connection test endpoint allows internal network scanning'
+Body: The POST /api/v1/connections/test endpoint will attempt a live connection to whatever host:port is supplied. An authenticated user can use this to probe internal network hosts. airflow/api_fastapi/execution_api/routes/connections.py accepts
+
+#104 | 'Scheduler: RCE via crafted serialized DAG in DagBag'
+Body: A DAG file containing a crafted __reduce__ method in a custom operator can trigger arbitrary code execution during DagBag parsing. File: airflow/dag_processing/processor.py BaseSerialization.deserialize()
+
+
+## Reporter roster (existing trackers mapped to reporter email)
+
+#102: b.researcher@secfirm.io
+
+## Incoming report
+
+<!-- SPDX-License-Identifier: Apache-2.0
+ https://www.apache.org/licenses/LICENSE-2.0 -->
+
+From: alice@example.com
+Subject: <PROJECT> REST API exposes DAG execution data without login
+
+I discovered that the Airflow REST API does not enforce authentication on the
+DAG runs endpoint. By sending a GET request to /api/v1/dags/my_dag/dagRuns
+with no Authorization header, I receive a full JSON response with task states,
+execution dates, and logs. This affects any Airflow deployment with the REST
+API enabled. Version tested: 2.9.3.
+
+
+Apply the semantic sweep and reporter-identity check. Return JSON only.
+"""
+
+
+def test_main_import_semantic_sweep_user_prompt_keeps_its_wording(capsys: pytest.CaptureFixture[str]):
+ """The semantic-sweep step renders its corpus, roster and report from its own template."""
+ assert _printed_user_prompt(capsys, _IMPORT_SWEEP_CASE) == (
+ _SPDX_HEADER + "\n\n" + _IMPORT_SWEEP_CASE_1_USER_PROMPT
+ )
+
+
# ---------------------------------------------------------------------------
# is_structural_expected
# ---------------------------------------------------------------------------