You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tooling.apache.org by sb...@apache.org on 2026/07/03 15:36:59 UTC

(tooling-trusted-releases) 01/02: Prefork task workers and stagger how many tasks they accept

This is an automated email from the ASF dual-hosted git repository.

sbp pushed a commit to branch sbp
in repository https://gitbox.apache.org/repos/asf/tooling-trusted-releases.git

commit f8e50f58d0053ed917b0b22f15287bcddc9deb0d
Author: Sean B. Palmer <se...@miscoranda.com>
AuthorDate: Fri Jul 3 16:25:12 2026 +0100

    Prefork task workers and stagger how many tasks they accept
---
 atr/constants.py   |  3 +++
 atr/manager.py     | 57 +++++++++++++++++++++++++++++++++++++++++++++++++-----
 atr/worker.py      |  2 +-
 playwright/test.py |  2 +-
 4 files changed, 57 insertions(+), 7 deletions(-)

diff --git a/atr/constants.py b/atr/constants.py
index 5060759a..6f90b886 100644
--- a/atr/constants.py
+++ b/atr/constants.py
@@ -258,3 +258,6 @@ INACTIVITY_WARNING_DAYS: Final[int] = 80
 # Fixed sub claim on JWTs minted from system PATs, and the UID we log for
 # their actions. The minting admin is recorded separately in PAT.created_by.
 SYSTEM_SERVICE_UID: Final[str] = "system"
+
+WORKER_MAX_TASKS_ENV: Final[str] = "ATR_WORKER_MAX_TASKS"
+WORKER_TASK_BUDGETS: Final[tuple[int, ...]] = (8, 14, 10, 16, 12)
diff --git a/atr/manager.py b/atr/manager.py
index 9d5ccec5..ea92d6e6 100644
--- a/atr/manager.py
+++ b/atr/manager.py
@@ -26,9 +26,11 @@ import os
 import signal
 import sys
 
+import sqlalchemy
 import sqlalchemy.engine as engine
 import sqlmodel
 
+import atr.constants as constants
 import atr.db as db
 import atr.log as log
 import atr.models.sql as sql
@@ -48,15 +50,15 @@ class WorkerManager:
     def __init__(
         self,
         min_workers: int = 4,
-        max_workers: int = 8,
         check_interval_seconds: float = 2.0,
         max_task_seconds: float = 300.0,
     ):
         self.min_workers = min_workers
-        self.max_workers = max_workers
+        self.max_workers = 2 * min_workers
         self.check_interval_seconds = check_interval_seconds
         self.max_task_seconds = max_task_seconds
         self.workers: dict[int, WorkerProcess] = {}
+        self.budget_index = 0
         self.running = False
         self.check_task: asyncio.Task | None = None
 
@@ -137,6 +139,10 @@ class WorkerManager:
             python_path = env.get("PYTHONPATH", "")
             env["PYTHONPATH"] = f"{project_root}:{python_path}" if python_path else project_root
 
+            budget = constants.WORKER_TASK_BUDGETS[self.budget_index % len(constants.WORKER_TASK_BUDGETS)]
+            self.budget_index += 1
+            env[constants.WORKER_MAX_TASKS_ENV] = str(budget)
+
             # Get absolute path to worker script
             worker_script = os.path.join(project_root, "atr", "worker.py")
 
@@ -168,10 +174,10 @@ class WorkerManager:
                 preexec_fn=os.setsid,
             )
 
-            worker = WorkerProcess(process, datetime.datetime.now(datetime.UTC))
+            worker = WorkerProcess(process, datetime.datetime.now(datetime.UTC), budget)
             if worker.pid:
                 self.workers[worker.pid] = worker
-                log.info(f"Started worker process {worker.pid}")
+                log.info(f"Started worker process {worker.pid} with task budget {budget}")
                 if global_worker_debug and log_file_path:
                     log.info(f"Worker {worker.pid} logs: {log_file_path}")
             else:
@@ -211,6 +217,9 @@ class WorkerManager:
                 # This also stops tasks if they have indeed been running for too long
                 if await self.check_task_duration(data, pid, worker):
                     exited_workers.append(pid)
+                    continue
+
+                await self.pre_spawn_replacement(data, pid, worker)
 
         # Remove exited workers
         for pid in exited_workers:
@@ -278,6 +287,42 @@ class WorkerManager:
             # TODO: Return False here to avoid over-reporting errors
             return False
 
+    async def pre_spawn_replacement(self, data: db.Session, pid: int, worker: WorkerProcess) -> None:
+        if worker.pre_spawned:
+            return
+        via = sql.validate_instrumented_attribute
+        try:
+            active_query = (
+                sqlmodel.select(sqlalchemy.func.count())
+                .select_from(sql.Task)
+                .where(
+                    sql.Task.pid == pid,
+                    sql.Task.status == sql.TaskStatus.ACTIVE,
+                    via(sql.Task.started) >= worker.started,
+                )
+            )
+            active = (await data.execute(active_query)).scalar_one()
+            if not active:
+                return
+            finished_query = (
+                sqlmodel.select(sqlalchemy.func.count())
+                .select_from(sql.Task)
+                .where(
+                    sql.Task.pid == pid,
+                    via(sql.Task.status).in_([sql.TaskStatus.COMPLETED, sql.TaskStatus.FAILED]),
+                    via(sql.Task.started) >= worker.started,
+                )
+            )
+            finished = (await data.execute(finished_query)).scalar_one()
+        except Exception as e:
+            log.error(f"Error counting processed tasks for worker {pid}: {e}")
+            return
+        if finished < (worker.budget - 1):
+            return
+        worker.pre_spawned = True
+        log.info(f"Worker {pid} started the last of its {worker.budget} tasks, spawning a replacement")
+        await self.spawn_worker()
+
     async def maintain_worker_pool(self) -> None:
         """Ensure we maintain the minimum number of workers."""
         current_count = len(self.workers)
@@ -350,10 +395,12 @@ class WorkerManager:
 class WorkerProcess:
     """Interface to control a worker process."""
 
-    def __init__(self, process: asyncio.subprocess.Process, started: datetime.datetime):
+    def __init__(self, process: asyncio.subprocess.Process, started: datetime.datetime, budget: int):
         self.process = process
         self.started = started
         self.last_checked = started
+        self.budget = budget
+        self.pre_spawned = False
 
     @property
     def pid(self) -> int | None:
diff --git a/atr/worker.py b/atr/worker.py
index 2c23f9ff..2373767b 100644
--- a/atr/worker.py
+++ b/atr/worker.py
@@ -367,7 +367,7 @@ def _truncate_single_line(text: str | None, max_length: int) -> str:
 async def _worker_loop_run() -> None:
     """Main worker loop."""
     processed = 0
-    max_to_process = 10
+    max_to_process = int(os.environ.get(constants.WORKER_MAX_TASKS_ENV, "10"))
     while True:
         log.clear_context()
         try:
diff --git a/playwright/test.py b/playwright/test.py
index 0831b38d..4bccd6e7 100755
--- a/playwright/test.py
+++ b/playwright/test.py
@@ -388,7 +388,7 @@ def poll_for_tasks_completion(page: Page, project_key: str, version_key: str, re
     polling_url = f"{ATR_BASE_URL}/admin/ongoing-tasks/{rev_path}"
     logging.info(f"Polling URL: {polling_url}")
 
-    max_wait_seconds = 60
+    max_wait_seconds = 120
     poll_interval_seconds = 0.5
     start_time = time.monotonic()
 


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org