You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tooling.apache.org by sb...@apache.org on 2026/07/03 15:36:59 UTC
(tooling-trusted-releases) 01/02: Prefork task workers and stagger how many tasks they accept
This is an automated email from the ASF dual-hosted git repository.
sbp pushed a commit to branch sbp
in repository https://gitbox.apache.org/repos/asf/tooling-trusted-releases.git
commit f8e50f58d0053ed917b0b22f15287bcddc9deb0d
Author: Sean B. Palmer <se...@miscoranda.com>
AuthorDate: Fri Jul 3 16:25:12 2026 +0100
Prefork task workers and stagger how many tasks they accept
---
atr/constants.py | 3 +++
atr/manager.py | 57 +++++++++++++++++++++++++++++++++++++++++++++++++-----
atr/worker.py | 2 +-
playwright/test.py | 2 +-
4 files changed, 57 insertions(+), 7 deletions(-)
diff --git a/atr/constants.py b/atr/constants.py
index 5060759a..6f90b886 100644
--- a/atr/constants.py
+++ b/atr/constants.py
@@ -258,3 +258,6 @@ INACTIVITY_WARNING_DAYS: Final[int] = 80
# Fixed sub claim on JWTs minted from system PATs, and the UID we log for
# their actions. The minting admin is recorded separately in PAT.created_by.
SYSTEM_SERVICE_UID: Final[str] = "system"
+
+WORKER_MAX_TASKS_ENV: Final[str] = "ATR_WORKER_MAX_TASKS"
+WORKER_TASK_BUDGETS: Final[tuple[int, ...]] = (8, 14, 10, 16, 12)
diff --git a/atr/manager.py b/atr/manager.py
index 9d5ccec5..ea92d6e6 100644
--- a/atr/manager.py
+++ b/atr/manager.py
@@ -26,9 +26,11 @@ import os
import signal
import sys
+import sqlalchemy
import sqlalchemy.engine as engine
import sqlmodel
+import atr.constants as constants
import atr.db as db
import atr.log as log
import atr.models.sql as sql
@@ -48,15 +50,15 @@ class WorkerManager:
def __init__(
self,
min_workers: int = 4,
- max_workers: int = 8,
check_interval_seconds: float = 2.0,
max_task_seconds: float = 300.0,
):
self.min_workers = min_workers
- self.max_workers = max_workers
+ self.max_workers = 2 * min_workers
self.check_interval_seconds = check_interval_seconds
self.max_task_seconds = max_task_seconds
self.workers: dict[int, WorkerProcess] = {}
+ self.budget_index = 0
self.running = False
self.check_task: asyncio.Task | None = None
@@ -137,6 +139,10 @@ class WorkerManager:
python_path = env.get("PYTHONPATH", "")
env["PYTHONPATH"] = f"{project_root}:{python_path}" if python_path else project_root
+ budget = constants.WORKER_TASK_BUDGETS[self.budget_index % len(constants.WORKER_TASK_BUDGETS)]
+ self.budget_index += 1
+ env[constants.WORKER_MAX_TASKS_ENV] = str(budget)
+
# Get absolute path to worker script
worker_script = os.path.join(project_root, "atr", "worker.py")
@@ -168,10 +174,10 @@ class WorkerManager:
preexec_fn=os.setsid,
)
- worker = WorkerProcess(process, datetime.datetime.now(datetime.UTC))
+ worker = WorkerProcess(process, datetime.datetime.now(datetime.UTC), budget)
if worker.pid:
self.workers[worker.pid] = worker
- log.info(f"Started worker process {worker.pid}")
+ log.info(f"Started worker process {worker.pid} with task budget {budget}")
if global_worker_debug and log_file_path:
log.info(f"Worker {worker.pid} logs: {log_file_path}")
else:
@@ -211,6 +217,9 @@ class WorkerManager:
# This also stops tasks if they have indeed been running for too long
if await self.check_task_duration(data, pid, worker):
exited_workers.append(pid)
+ continue
+
+ await self.pre_spawn_replacement(data, pid, worker)
# Remove exited workers
for pid in exited_workers:
@@ -278,6 +287,42 @@ class WorkerManager:
# TODO: Return False here to avoid over-reporting errors
return False
+ async def pre_spawn_replacement(self, data: db.Session, pid: int, worker: WorkerProcess) -> None:
+ if worker.pre_spawned:
+ return
+ via = sql.validate_instrumented_attribute
+ try:
+ active_query = (
+ sqlmodel.select(sqlalchemy.func.count())
+ .select_from(sql.Task)
+ .where(
+ sql.Task.pid == pid,
+ sql.Task.status == sql.TaskStatus.ACTIVE,
+ via(sql.Task.started) >= worker.started,
+ )
+ )
+ active = (await data.execute(active_query)).scalar_one()
+ if not active:
+ return
+ finished_query = (
+ sqlmodel.select(sqlalchemy.func.count())
+ .select_from(sql.Task)
+ .where(
+ sql.Task.pid == pid,
+ via(sql.Task.status).in_([sql.TaskStatus.COMPLETED, sql.TaskStatus.FAILED]),
+ via(sql.Task.started) >= worker.started,
+ )
+ )
+ finished = (await data.execute(finished_query)).scalar_one()
+ except Exception as e:
+ log.error(f"Error counting processed tasks for worker {pid}: {e}")
+ return
+ if finished < (worker.budget - 1):
+ return
+ worker.pre_spawned = True
+ log.info(f"Worker {pid} started the last of its {worker.budget} tasks, spawning a replacement")
+ await self.spawn_worker()
+
async def maintain_worker_pool(self) -> None:
"""Ensure we maintain the minimum number of workers."""
current_count = len(self.workers)
@@ -350,10 +395,12 @@ class WorkerManager:
class WorkerProcess:
"""Interface to control a worker process."""
- def __init__(self, process: asyncio.subprocess.Process, started: datetime.datetime):
+ def __init__(self, process: asyncio.subprocess.Process, started: datetime.datetime, budget: int):
self.process = process
self.started = started
self.last_checked = started
+ self.budget = budget
+ self.pre_spawned = False
@property
def pid(self) -> int | None:
diff --git a/atr/worker.py b/atr/worker.py
index 2c23f9ff..2373767b 100644
--- a/atr/worker.py
+++ b/atr/worker.py
@@ -367,7 +367,7 @@ def _truncate_single_line(text: str | None, max_length: int) -> str:
async def _worker_loop_run() -> None:
"""Main worker loop."""
processed = 0
- max_to_process = 10
+ max_to_process = int(os.environ.get(constants.WORKER_MAX_TASKS_ENV, "10"))
while True:
log.clear_context()
try:
diff --git a/playwright/test.py b/playwright/test.py
index 0831b38d..4bccd6e7 100755
--- a/playwright/test.py
+++ b/playwright/test.py
@@ -388,7 +388,7 @@ def poll_for_tasks_completion(page: Page, project_key: str, version_key: str, re
polling_url = f"{ATR_BASE_URL}/admin/ongoing-tasks/{rev_path}"
logging.info(f"Polling URL: {polling_url}")
- max_wait_seconds = 60
+ max_wait_seconds = 120
poll_interval_seconds = 0.5
start_time = time.monotonic()
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org