You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@magpie.apache.org by po...@apache.org on 2026/10/18 18:28:33 UTC
(magpie) branch main updated: test(adversarial-review): give the timing tests room to start their stubs (#1699)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/magpie.git
The following commit(s) were added to refs/heads/main by this push:
new 77e81450 test(adversarial-review): give the timing tests room to start their stubs (#1699)
77e81450 is described below
commit 77e81450ddd62a8b9941f1c7a50c390d09606b68
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Sun Oct 18 20:28:26 2026 +0200
test(adversarial-review): give the timing tests room to start their stubs (#1699)
* test(adversarial-review): give the answering reviewer time to start
test_one_timeout_does_not_block_the_others ran both stub reviewers with a
one-second budget, and the one meant to answer is a Python process that
has to start first. On a loaded machine (the parallel pre-commit pytest
run) start-up alone took longer, so the answering reviewer timed out too
and the test failed. Give it five seconds; the other stub still sleeps
for a minute, so the timeout case is unchanged.
Generated-by: Claude Opus 5
* test(adversarial-review): widen the other timing margins
Two more runner tests measured wall-clock time against bounds that left
little room for starting the Python stubs, and one failed under the
parallel pre-commit pytest run (3.03 s against 2.8 s):
- the parallel run sleeps 3 s per reviewer and must finish under 5.5 s
(6 s one after the other), not 1.5 s under 2.8 s;
- a finished answer with a lingering helper must arrive under 8 s with a
10 s timeout, not under 4 s with 5 s.
Each test still tells the behaviour apart from its failure mode.
Generated-by: Claude Opus 5
---
tools/adversarial-review/tests/test_runner.py | 17 +++++++++++------
1 file changed, 11 insertions(+), 6 deletions(-)
diff --git a/tools/adversarial-review/tests/test_runner.py b/tools/adversarial-review/tests/test_runner.py
index 5f7bc361..380f1c79 100644
--- a/tools/adversarial-review/tests/test_runner.py
+++ b/tools/adversarial-review/tests/test_runner.py
@@ -104,15 +104,17 @@ def test_timeout_kills_the_whole_process_group(stub_bin, tmp_path):
def test_reviewers_run_in_parallel_and_keep_order(stub_bin, tmp_path):
bin_dir, make = stub_bin
- make("codex", "import time; time.sleep(1.5)\n" + CODEX_OK)
+ make("codex", "import time; time.sleep(3)\n" + CODEX_OK)
make(
"gemini",
- "import json, time; time.sleep(1.5)\nprint(json.dumps({'response': json.dumps({'findings': []})}))\n",
+ "import json, time; time.sleep(3)\nprint(json.dumps({'response': json.dumps({'findings': []})}))\n",
)
contexts = {n: ctx(tmp_path, n) for n in ("gemini", "codex")}
start = time.monotonic()
results = run_all(["gemini", "codex"], contexts, 30, {"PATH": str(bin_dir)})
- assert time.monotonic() - start < 2.8
+ # One after the other they take 6 s; the 2.5 s left over is for starting
+ # both stubs on a loaded machine (a 1.3 s margin failed under pre-commit).
+ assert time.monotonic() - start < 5.5
assert [(r.reviewer, r.status) for r in results] == [("gemini", "ok"), ("codex", "ok")]
@@ -121,7 +123,9 @@ def test_one_timeout_does_not_block_the_others(stub_bin, tmp_path):
make("codex", CODEX_OK)
make("gemini", "import time; time.sleep(60)")
contexts = {n: ctx(tmp_path, n) for n in ("codex", "gemini")}
- results = {r.reviewer: r for r in run_all(["codex", "gemini"], contexts, 1, {"PATH": str(bin_dir)})}
+ # Room for codex's stub to start and answer on a loaded machine (a 1 s
+ # budget failed under the parallel pre-commit run); gemini sleeps far longer.
+ results = {r.reviewer: r for r in run_all(["codex", "gemini"], contexts, 5, {"PATH": str(bin_dir)})}
assert results["codex"].status == "ok" and results["gemini"].status == "timeout"
@@ -136,9 +140,10 @@ def test_a_finished_answer_is_kept_when_a_helper_lingers(stub_bin, tmp_path):
f"print(json.dumps({{'findings': [{FINDING!r}]}}))\n",
)
start = time.monotonic()
- r = run_one("copilot", ctx(tmp_path, "copilot"), 5, {"PATH": str(bin_dir)})
+ r = run_one("copilot", ctx(tmp_path, "copilot"), 10, {"PATH": str(bin_dir)})
assert r.status == "ok" and len(r.findings) == 1
- assert time.monotonic() - start < 4
+ # Well inside the timeout, which a held stdout would have run into.
+ assert time.monotonic() - start < 8
def test_timeout_is_bounded_even_when_a_helper_escapes_the_group(stub_bin, tmp_path):