You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tooling.apache.org by ak...@apache.org on 2026/06/16 03:21:05 UTC
(tooling-agents) 07/14: asvs_job_runner: fix retry deadlock and summary miscount
This is an automated email from the ASF dual-hosted git repository.
andrewmusselman pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tooling-agents.git
commit 77fc7f2136d0e1080f0dfd0dfce73afa6078a916
Author: Andrew Musselman <ak...@apache.org>
AuthorDate: Sun Jun 7 06:18:47 2026 +0000
asvs_job_runner: fix retry deadlock and summary miscount
run_job recursed into itself while holding self.semaphore and slept during
backoff inside the held semaphore. asyncio.Semaphore is not reentrant, so
at concurrency >= max_concurrent, N simultaneously-retrying jobs each held
a slot while awaiting a slot -> deadlock; and a backing-off job uselessly
occupied a slot for up to 60s. Replace the recursion with an internal
retry loop that holds the semaphore only for each attempt and sleeps after
releasing it.
Also tighten run_all's summary: it folded skipped-done into 'succeeded'
and miscounted raised exceptions. Count each bucket explicitly.
---
ASVS/agents/asvs_job_runner.py | 137 ++++++++++++++++++++++++-----------------
1 file changed, 79 insertions(+), 58 deletions(-)
diff --git a/ASVS/agents/asvs_job_runner.py b/ASVS/agents/asvs_job_runner.py
index 3d7a109..dc978f4 100644
--- a/ASVS/agents/asvs_job_runner.py
+++ b/ASVS/agents/asvs_job_runner.py
@@ -156,49 +156,72 @@ class JobRunner:
)
return {"status": "skipped-fatal", "error": state.get("last_error")}
- # Claim the job
- attempts = (state.get("attempts", 0) if state else 0) + 1
- async with self.semaphore:
- self._write_state(
- key,
- self.RUNNING,
- attempts=attempts,
- started_at=self._now(),
- last_error=None,
- )
-
- try:
- result = await callable_()
+ # Claim the job. Retry as an internal loop (NOT recursion): the
+ # old code recursed into run_job while still holding self.semaphore
+ # and slept during backoff inside the held semaphore. Both are bugs
+ # at concurrency >= max_concurrent: asyncio.Semaphore is not
+ # reentrant, so N jobs all retrying would each hold a slot while
+ # waiting for a slot -> deadlock; and a backing-off job needlessly
+ # occupied a slot for up to 60s. Now the semaphore is held only for
+ # the attempt itself, and the backoff sleep happens after release.
+ attempts = (state.get("attempts", 0) if state else 0)
+
+ while True:
+ attempts += 1
+
+ async with self.semaphore:
self._write_state(
key,
- self.DONE,
- completed_at=self._now(),
- output_uri=(
- result.get("output_uri")
- if isinstance(result, dict)
- else None
- ),
+ self.RUNNING,
+ attempts=attempts,
+ started_at=self._now(),
+ last_error=None,
)
- print(
- f"[runner] DONE {key} (attempt {attempts})",
- flush=True,
- )
- return result
-
- except Exception as e:
- err_type = type(e).__name__
- err_msg = str(e) or "(no message)"
- is_rate_limit = "RateLimitError" in err_type or "429" in err_msg
- is_timeout = "Timeout" in err_type
- retryable = is_rate_limit or is_timeout or attempts < self.MAX_ATTEMPTS
-
- if retryable and attempts < self.MAX_ATTEMPTS:
+ try:
+ result = await callable_()
+ self._write_state(
+ key,
+ self.DONE,
+ completed_at=self._now(),
+ output_uri=(
+ result.get("output_uri")
+ if isinstance(result, dict)
+ else None
+ ),
+ )
+ print(
+ f"[runner] DONE {key} (attempt {attempts})",
+ flush=True,
+ )
+ return result
+
+ except Exception as e:
+ err_type = type(e).__name__
+ err_msg = str(e) or "(no message)"
+ is_rate_limit = "RateLimitError" in err_type or "429" in err_msg
+
+ if attempts >= self.MAX_ATTEMPTS:
+ self._write_state(
+ key,
+ self.FAILED_FATAL,
+ last_error=f"{err_type}: {err_msg}",
+ completed_at=self._now(),
+ )
+ print(
+ f"[runner] FATAL {key} after {attempts} "
+ f"attempt(s): {err_type}: {err_msg[:120]}",
+ flush=True,
+ )
+ return {"status": "failed-fatal", "error": err_msg}
+
+ # Mark for retry; compute backoff. The sleep happens
+ # AFTER the semaphore is released (below), so a
+ # backing-off job does not hold a concurrency slot.
self._write_state(
key,
self.QUEUED,
last_error=f"{err_type}: {err_msg}",
)
- # Backoff before the caller might retry
backoff = min(60, 2 ** attempts)
if is_rate_limit:
backoff = max(backoff, 30)
@@ -208,22 +231,9 @@ class JobRunner:
f"{err_type}: {err_msg[:120]}",
flush=True,
)
- await asyncio.sleep(backoff)
- # Recurse — the same call attempts the job again
- return await self.run_job(component, phase, callable_, section)
- else:
- self._write_state(
- key,
- self.FAILED_FATAL,
- last_error=f"{err_type}: {err_msg}",
- completed_at=self._now(),
- )
- print(
- f"[runner] FATAL {key} after {attempts} "
- f"attempt(s): {err_type}: {err_msg[:120]}",
- flush=True,
- )
- return {"status": "failed-fatal", "error": err_msg}
+
+ # Semaphore released. Back off without holding a slot, then loop.
+ await asyncio.sleep(backoff)
# ----- Bulk run -----
@@ -250,16 +260,27 @@ class JobRunner:
results = await asyncio.gather(*tasks, return_exceptions=True)
- # Print summary
- succeeded = sum(1 for r in results if isinstance(r, dict) and r.get("status") not in ("failed-fatal", "skipped-fatal"))
- failed = sum(1 for r in results if isinstance(r, dict) and r.get("status") in ("failed-fatal",))
- skipped_done = sum(1 for r in results if isinstance(r, dict) and r.get("status") == "skipped-done")
- skipped_fatal = sum(1 for r in results if isinstance(r, dict) and r.get("status") == "skipped-fatal")
+ # Print summary. Count each bucket explicitly; do NOT fold skips
+ # into success (the old `not in (failed-fatal, skipped-fatal)`
+ # counted skipped-done as a fresh success and miscounted raised
+ # exceptions). A successful phase result is a dict without a
+ # failure/skip status.
+ def _status(r):
+ if isinstance(r, dict):
+ return r.get("status")
+ return "exception"
+
+ completed = sum(1 for r in results if _status(r) in (None, "ok"))
+ failed = sum(1 for r in results if _status(r) == "failed-fatal")
+ skipped_done = sum(1 for r in results if _status(r) == "skipped-done")
+ skipped_fatal = sum(1 for r in results if _status(r) == "skipped-fatal")
+ raised = sum(1 for r in results if _status(r) == "exception")
print(
- f"[runner] run complete: {succeeded} succeeded, "
+ f"[runner] run complete: {completed} completed, "
f"{failed} fatal, "
f"{skipped_done} skipped (already done), "
- f"{skipped_fatal} skipped (previously fatal)",
+ f"{skipped_fatal} skipped (previously fatal), "
+ f"{raised} raised",
flush=True,
)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org