You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tooling.apache.org by sb...@apache.org on 2026/07/28 16:50:45 UTC
(tooling-trusted-releases) branch sbp updated: Add a broken status to indicate tasks which may be retried
This is an automated email from the ASF dual-hosted git repository.
sbp pushed a commit to branch sbp
in repository https://gitbox.apache.org/repos/asf/tooling-trusted-releases.git
The following commit(s) were added to refs/heads/sbp by this push:
new 3570b0e7 Add a broken status to indicate tasks which may be retried
3570b0e7 is described below
commit 3570b0e7d40089b848844e2ba5e61abe72014108
Author: Sean B. Palmer <se...@miscoranda.com>
AuthorDate: Tue Jul 28 15:59:16 2026 +0100
Add a broken status to indicate tasks which may be retried
---
atr/admin/__init__.py | 1 +
atr/db/interaction.py | 2 +-
atr/manager.py | 4 +-
atr/models/sql.py | 7 +-
atr/tasks/task.py | 10 +-
atr/worker.py | 15 ++-
migrations/versions/0113_2026.07.28_dd069840.py | 138 ++++++++++++++++++++++++
tests/unit/test_worker_defer.py | 27 +++--
8 files changed, 188 insertions(+), 16 deletions(-)
diff --git a/atr/admin/__init__.py b/atr/admin/__init__.py
index 5c456a67..29328388 100644
--- a/atr/admin/__init__.py
+++ b/atr/admin/__init__.py
@@ -1927,6 +1927,7 @@ async def tasks_recent(_session: web.Committer, _tasks_recent: Literal["tasks/re
sql.TaskStatus.ACTIVE: ".table-info",
sql.TaskStatus.COMPLETED: ".table-success",
sql.TaskStatus.FAILED: ".table-danger",
+ sql.TaskStatus.BROKEN: ".table-warning",
}.get(task.status, "")
if (task.started is not None) and (task.completed is not None):
took_seconds = (task.completed - task.started).total_seconds()
diff --git a/atr/db/interaction.py b/atr/db/interaction.py
index 46911de4..59b607d1 100644
--- a/atr/db/interaction.py
+++ b/atr/db/interaction.py
@@ -1076,7 +1076,7 @@ async def wait_for_task(
return False
for _attempt in range(timeout_s * 10):
await data.refresh(t)
- if t.status == sql.TaskStatus.FAILED:
+ if t.status in (sql.TaskStatus.FAILED, sql.TaskStatus.BROKEN):
raise InteractionError(f"Task {task.id} failed with error {t.error}")
if t.status == desired_status:
return True
diff --git a/atr/manager.py b/atr/manager.py
index ab450bbe..1b2e64cb 100644
--- a/atr/manager.py
+++ b/atr/manager.py
@@ -325,7 +325,9 @@ class WorkerManager:
.select_from(sql.Task)
.where(
sql.Task.pid == pid,
- via(sql.Task.status).in_([sql.TaskStatus.COMPLETED, sql.TaskStatus.FAILED]),
+ via(sql.Task.status).in_(
+ [sql.TaskStatus.COMPLETED, sql.TaskStatus.FAILED, sql.TaskStatus.BROKEN]
+ ),
via(sql.Task.started) >= worker.started,
)
)
diff --git a/atr/models/sql.py b/atr/models/sql.py
index 49a318fd..1adb4ff0 100644
--- a/atr/models/sql.py
+++ b/atr/models/sql.py
@@ -227,6 +227,7 @@ class TaskStatus(enum.StrEnum):
ACTIVE = "active"
COMPLETED = "completed"
FAILED = "failed"
+ BROKEN = "broken"
class TaskType(enum.StrEnum):
@@ -783,8 +784,8 @@ class Task(sqlmodel.SQLModel, table=True):
sqlalchemy.Index("ix_task_status_added", "status", "added"),
# Ensure valid status transitions:
# - QUEUED can transition to ACTIVE
- # - ACTIVE can transition to COMPLETED or FAILED
- # - COMPLETED and FAILED are terminal states
+ # - ACTIVE can transition to COMPLETED, FAILED, or BROKEN
+ # - COMPLETED, FAILED, and BROKEN are terminal states
sqlalchemy.CheckConstraint(
"""
(
@@ -796,6 +797,8 @@ class Task(sqlmodel.SQLModel, table=True):
OR (status = 'COMPLETED' AND completed IS NOT NULL AND result IS NOT NULL)
-- ACTIVE -> FAILED requires setting completed time and error (result optional)
OR (status = 'FAILED' AND completed IS NOT NULL AND error IS NOT NULL)
+ -- ACTIVE -> BROKEN requires setting completed time and error (result optional)
+ OR (status = 'BROKEN' AND completed IS NOT NULL AND error IS NOT NULL)
)
""",
name="valid_task_status_transitions",
diff --git a/atr/tasks/task.py b/atr/tasks/task.py
index db64a768..62bc26ae 100644
--- a/atr/tasks/task.py
+++ b/atr/tasks/task.py
@@ -17,7 +17,7 @@
from __future__ import annotations
-from typing import Final
+from typing import Any, Final
import atr.constants as constants
import atr.log as log
@@ -29,6 +29,7 @@ QUEUED: Final = sql.TaskStatus.QUEUED
ACTIVE: Final = sql.TaskStatus.ACTIVE
COMPLETED: Final = sql.TaskStatus.COMPLETED
FAILED: Final = sql.TaskStatus.FAILED
+BROKEN: Final = sql.TaskStatus.BROKEN
# The recurring tasks reschedule themselves and carry no lasting per-run value, so on
# success they are logged to a file and dropped rather than kept in the table.
@@ -50,6 +51,13 @@ TASK_TYPE_TIMEOUT_SECONDS: Final[dict[sql.TaskType, int]] = {
}
+class CheckRetryableError(Exception):
+ def __init__(self, message: str, data: Any = None) -> None:
+ super().__init__(message)
+ self.message = message
+ self.data = data
+
+
class DeferredError(Exception):
"""Raised by a handler to send its own task back to the queue for a later attempt."""
diff --git a/atr/worker.py b/atr/worker.py
index 97190bc9..7896be1b 100644
--- a/atr/worker.py
+++ b/atr/worker.py
@@ -190,10 +190,6 @@ def _setup_logging() -> None:
)
-def _task_completed_log(record: dict[str, Any]) -> None:
- logging.getLogger(_TASK_LOG_LOGGER).info(json.dumps(record, allow_nan=False))
-
-
def _task_args_for_log(
task_type: str | sql.TaskType, task_args: list[str] | dict[str, Any]
) -> list[str] | dict[str, Any]:
@@ -204,6 +200,10 @@ def _task_args_for_log(
}
+def _task_completed_log(record: dict[str, Any]) -> None:
+ logging.getLogger(_TASK_LOG_LOGGER).info(json.dumps(record, allow_nan=False))
+
+
def _task_completed_record(
task_obj: sql.Task, result: results.Results | None, completed: datetime.datetime
) -> dict[str, Any]:
@@ -346,6 +346,11 @@ async def _task_process(task_id: int, task_type: str, task_args: list[str] | dic
log.info(f"Task {task_id} ({task_type}) deferred, re-queued for a later attempt")
await _task_defer(task_id)
return
+ except task.CheckRetryableError as e:
+ task_results = None
+ status = task.BROKEN
+ log.error(f"Task {task_id} failed with a retryable error: {e}")
+ error = str(e)
except Exception as e:
task_results = None
status = task.FAILED
@@ -377,7 +382,7 @@ async def _task_result_process(
task_obj.completed = completed
task_obj.result = task_results
- if status == task.FAILED:
+ if status in (task.FAILED, task.BROKEN):
normalised_error = ((error or "").strip()) or "unknown error"
task_obj.error = normalised_error
notify_args = (
diff --git a/migrations/versions/0113_2026.07.28_dd069840.py b/migrations/versions/0113_2026.07.28_dd069840.py
new file mode 100644
index 00000000..5c55cdaf
--- /dev/null
+++ b/migrations/versions/0113_2026.07.28_dd069840.py
@@ -0,0 +1,138 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Add the BROKEN task status for retryable failures
+
+Revision ID: 0113_2026.07.28_dd069840
+Revises: 0112_2026.07.22_d374db02
+Create Date: 2026-07-28 13:34:48.150920+00:00
+"""
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+from alembic import op
+
+# Revision identifiers, used by Alembic
+revision: str = "0113_2026.07.28_dd069840"
+down_revision: str | None = "0112_2026.07.22_d374db02"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+_CHECK_WITHOUT_BROKEN = """
+ (
+ -- Initial state is always valid
+ status = 'QUEUED'
+ -- QUEUED -> ACTIVE requires setting started time and pid
+ OR (status = 'ACTIVE' AND started IS NOT NULL AND pid IS NOT NULL)
+ -- ACTIVE -> COMPLETED requires setting completed time and result
+ OR (status = 'COMPLETED' AND completed IS NOT NULL AND result IS NOT NULL)
+ -- ACTIVE -> FAILED requires setting completed time and error (result optional)
+ OR (status = 'FAILED' AND completed IS NOT NULL AND error IS NOT NULL)
+ )
+ """
+
+_CHECK_WITH_BROKEN = """
+ (
+ -- Initial state is always valid
+ status = 'QUEUED'
+ -- QUEUED -> ACTIVE requires setting started time and pid
+ OR (status = 'ACTIVE' AND started IS NOT NULL AND pid IS NOT NULL)
+ -- ACTIVE -> COMPLETED requires setting completed time and result
+ OR (status = 'COMPLETED' AND completed IS NOT NULL AND result IS NOT NULL)
+ -- ACTIVE -> FAILED requires setting completed time and error (result optional)
+ OR (status = 'FAILED' AND completed IS NOT NULL AND error IS NOT NULL)
+ -- ACTIVE -> BROKEN requires setting completed time and error (result optional)
+ OR (status = 'BROKEN' AND completed IS NOT NULL AND error IS NOT NULL)
+ )
+ """
+
+
+def upgrade() -> None:
+ # Deployed databases lost the valid_task_status_transitions constraint in June 2025:
+ # migration 0007 rebuilt the task table under SQLAlchemy 2.0.40, whose SQLite
+ # reflection could not parse the multi-line CHECK, so the rebuild silently recreated
+ # the table without it. The batch operations here therefore declare the table
+ # explicitly instead of reflecting it, which handles databases with the constraint
+ # present, absent, or unparseable, and rewrites all of them to the same schema.
+ #
+ # Rows written while the constraint was absent are normalised first, so that the
+ # rebuild's data copy cannot fail against the restored constraint.
+ op.execute(
+ "UPDATE task SET completed = COALESCE(completed, started, added)"
+ " WHERE status IN ('COMPLETED', 'FAILED') AND completed IS NULL"
+ )
+ op.execute("UPDATE task SET result = 'null' WHERE status = 'COMPLETED' AND result IS NULL")
+ op.execute("UPDATE task SET error = 'unknown error' WHERE status = 'FAILED' AND error IS NULL")
+ # An ACTIVE row with no recorded worker cannot be requeued safely, because its task
+ # may already have had external effects, so it is failed as unconfirmed instead
+ op.execute(
+ "UPDATE task SET status = 'FAILED', completed = COALESCE(completed, started, added),"
+ " error = 'Task was active with no recorded worker'"
+ " WHERE status = 'ACTIVE' AND (started IS NULL OR pid IS NULL)"
+ )
+ with op.batch_alter_table("task", schema=None, copy_from=_task_table()) as batch_op:
+ batch_op.create_check_constraint(
+ batch_op.f("ck_task_valid_task_status_transitions"),
+ _CHECK_WITH_BROKEN,
+ )
+
+
+def downgrade() -> None:
+ op.execute("UPDATE task SET status = 'FAILED' WHERE status = 'BROKEN'")
+ with op.batch_alter_table("task", schema=None, copy_from=_task_table()) as batch_op:
+ batch_op.create_check_constraint(
+ batch_op.f("ck_task_valid_task_status_transitions"),
+ _CHECK_WITHOUT_BROKEN,
+ )
+
+
+def _task_table() -> sa.Table:
+ # The physical task schema as of 0112, transcribed rather than reflected, and
+ # carrying no CHECK constraint so that the batch rebuild replaces whatever a given
+ # database has with exactly the constraint created above
+ return sa.Table(
+ "task",
+ sa.MetaData(),
+ sa.Column("id", sa.Integer(), nullable=False),
+ sa.Column("status", sa.String(9), nullable=False),
+ sa.Column("task_type", sa.String(23), nullable=False),
+ sa.Column("task_args", sa.JSON(), nullable=True),
+ sa.Column("added", sa.TIMESTAMP(), nullable=False),
+ sa.Column("started", sa.TIMESTAMP(), nullable=True),
+ sa.Column("pid", sa.Integer(), nullable=True),
+ sa.Column("completed", sa.TIMESTAMP(), nullable=True),
+ sa.Column("result", sa.JSON(), nullable=True),
+ sa.Column("error", sa.String(), nullable=True),
+ sa.Column("revision_number", sa.String(), nullable=True),
+ sa.Column("primary_rel_path", sa.String(), nullable=True),
+ sa.Column("asf_uid", sa.String(), server_default=sa.text("('')"), nullable=False),
+ sa.Column("scheduled", sa.TIMESTAMP(), nullable=True),
+ sa.Column("inputs_hash", sa.String(), nullable=True),
+ sa.Column("project_key", sa.String(), nullable=True),
+ sa.Column("version_key", sa.String(), nullable=True),
+ sa.PrimaryKeyConstraint("id", name="pk_task"),
+ sa.ForeignKeyConstraint(["project_key"], ["project.key"], name="fk_task_project_key_project"),
+ sa.Index("ix_task_added", "added"),
+ sa.Index("ix_task_inputs_hash", "inputs_hash", unique=True),
+ sa.Index("ix_task_primary_rel_path", "primary_rel_path"),
+ sa.Index("ix_task_revision_number", "revision_number"),
+ sa.Index("ix_task_scheduled", "scheduled"),
+ sa.Index("ix_task_status", "status"),
+ sa.Index("ix_task_status_added", "status", "added"),
+ sa.Index("ix_task_version_key", "version_key"),
+ )
diff --git a/tests/unit/test_worker_defer.py b/tests/unit/test_worker_defer.py
index 3735d0dc..7c98e748 100644
--- a/tests/unit/test_worker_defer.py
+++ b/tests/unit/test_worker_defer.py
@@ -25,6 +25,12 @@ import atr.tasks.task as task
import atr.worker as worker
+def test_task_args_for_log_leaves_other_tasks_unchanged() -> None:
+ task_args = {"body": "Not a mail body"}
+
+ assert worker._task_args_for_log(sql.TaskType.MAINTENANCE, task_args) is task_args
+
+
def test_task_args_for_log_redacts_message_body_and_recipients() -> None:
task_args = {
"email_sender": "sender@apache.org",
@@ -54,12 +60,6 @@ def test_task_args_for_log_redacts_message_body_and_recipients() -> None:
assert task_args["body"] == "Secret body"
-def test_task_args_for_log_leaves_other_tasks_unchanged() -> None:
- task_args = {"body": "Not a mail body"}
-
- assert worker._task_args_for_log(sql.TaskType.MAINTENANCE, task_args) is task_args
-
-
async def test_task_process_defers_when_ldap_unavailable(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr("atr.config.is_production_mode", lambda: True)
monkeypatch.setattr("atr.ldap.account_lookup", mock.AsyncMock(side_effect=ldap.UnavailableError("down")))
@@ -90,3 +90,18 @@ async def test_task_process_fails_when_handler_ldap_unavailable(monkeypatch: pyt
defer.assert_not_awaited()
result_process.assert_awaited_once_with(1, None, task.FAILED, "down")
+
+
+async def test_task_process_marks_retryable_check_failures_broken(monkeypatch: pytest.MonkeyPatch) -> None:
+ async def handler(args: object) -> None:
+ raise task.CheckRetryableError("clone failed", {"repo_url": "https://example.invalid/repo.git"})
+
+ monkeypatch.setattr("atr.config.is_production_mode", lambda: False)
+ monkeypatch.setattr("atr.config.is_ldap_configured", lambda: False)
+ monkeypatch.setattr("atr.tasks.resolve", lambda task_type: handler)
+ result_process = mock.AsyncMock()
+ monkeypatch.setattr("atr.worker._task_result_process", result_process)
+
+ await worker._task_process(1, sql.TaskType.COMPARE_SOURCE_TREES.value, {}, "alice")
+
+ result_process.assert_awaited_once_with(1, None, task.BROKEN, "clone failed")
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org