You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by go...@apache.org on 2025/07/29 08:15:23 UTC

(airflow) branch v3-0-test updated: Make BaseOperator on_kill functionality work with TaskSDK (#53718) (#53832)

This is an automated email from the ASF dual-hosted git repository.

gopidesu pushed a commit to branch v3-0-test
in repository https://gitbox.apache.org/repos/asf/airflow.git


The following commit(s) were added to refs/heads/v3-0-test by this push:
     new 400275fce5f Make BaseOperator on_kill functionality work with TaskSDK (#53718) (#53832)
400275fce5f is described below

commit 400275fce5f749df2a98b22f9de93bf3bcbf8d25
Author: GPK <go...@gmail.com>
AuthorDate: Tue Jul 29 09:15:15 2025 +0100

    Make BaseOperator on_kill functionality work with TaskSDK (#53718) (#53832)
    
    * Make BaseOperator on_kill functionality work with TaskSDK
    
    * Fix static checks
    
    * Resolve review comments
    
    * Update task-sdk/tests/task_sdk/execution_time/test_supervisor.py
    
    
    
    ---------
    
    Co-authored-by: Ash Berlin-Taylor <as...@firemirror.com>
---
 .../src/airflow/sdk/execution_time/task_runner.py  | 13 ++++
 .../task_sdk/execution_time/test_supervisor.py     | 81 ++++++++++++++++++++++
 2 files changed, 94 insertions(+)

diff --git a/task-sdk/src/airflow/sdk/execution_time/task_runner.py b/task-sdk/src/airflow/sdk/execution_time/task_runner.py
index bdd3b240daf..62aa7d37b7f 100644
--- a/task-sdk/src/airflow/sdk/execution_time/task_runner.py
+++ b/task-sdk/src/airflow/sdk/execution_time/task_runner.py
@@ -832,6 +832,8 @@ def run(
     log: Logger,
 ) -> tuple[TaskInstanceState, ToSupervisor | None, BaseException | None]:
     """Run the task in this process."""
+    import signal
+
     from airflow.exceptions import (
         AirflowException,
         AirflowFailException,
@@ -849,6 +851,17 @@ def run(
         assert ti.task is not None
         assert isinstance(ti.task, BaseOperator)
 
+    parent_pid = os.getpid()
+
+    def _on_term(signum, frame):
+        pid = os.getpid()
+        if pid != parent_pid:
+            return
+
+        ti.task.on_kill()
+
+    signal.signal(signal.SIGTERM, _on_term)
+
     msg: ToSupervisor | None = None
     state: TaskInstanceState
     error: BaseException | None = None
diff --git a/task-sdk/tests/task_sdk/execution_time/test_supervisor.py b/task-sdk/tests/task_sdk/execution_time/test_supervisor.py
index 27b655a323d..d1109b68e4e 100644
--- a/task-sdk/tests/task_sdk/execution_time/test_supervisor.py
+++ b/task-sdk/tests/task_sdk/execution_time/test_supervisor.py
@@ -44,6 +44,7 @@ from task_sdk import FAKE_BUNDLE, make_client
 from uuid6 import uuid7
 
 from airflow.executors.workloads import BundleInfo
+from airflow.sdk import BaseOperator
 from airflow.sdk.api import client as sdk_client
 from airflow.sdk.api.client import ServerResponseError
 from airflow.sdk.api.datamodels._generated import (
@@ -113,6 +114,7 @@ from airflow.sdk.execution_time.supervisor import (
     set_supervisor_comms,
     supervise,
 )
+from airflow.sdk.execution_time.task_runner import run
 from airflow.utils import timezone, timezone as tz
 
 if TYPE_CHECKING:
@@ -330,6 +332,85 @@ class TestWatchedSubprocess:
             ]
         )
 
+    def test_on_kill_hook_called_when_sigkilled(
+        self,
+        client_with_ti_start,
+        mocked_parse,
+        make_ti_context,
+        mock_supervisor_comms,
+        create_runtime_ti,
+        make_ti_context_dict,
+        capfd,
+    ):
+        main_pid = os.getpid()
+        ti_id = "4d828a62-a417-4936-a7a6-2b3fabacecab"
+
+        def handle_request(request: httpx.Request) -> httpx.Response:
+            if request.url.path == f"/task-instances/{ti_id}/heartbeat":
+                return httpx.Response(
+                    status_code=409,
+                    json={
+                        "detail": {
+                            "reason": "not_running",
+                            "message": "TI is no longer in the 'running' state. Task state might be externally set and task should terminate",
+                            "current_state": "failed",
+                        }
+                    },
+                )
+            if request.url.path == f"/task-instances/{ti_id}/run":
+                return httpx.Response(200, json=make_ti_context_dict())
+            return httpx.Response(status_code=204)
+
+        def subprocess_main():
+            # Ensure we follow the "protocol" and get the startup message before we do anything
+            CommsDecoder()._get_response()
+
+            class CustomOperator(BaseOperator):
+                def execute(self, context):
+                    for i in range(1000):
+                        print(f"Iteration {i}")
+                        sleep(1)
+
+                def on_kill(self) -> None:
+                    print("On kill hook called!")
+
+            task = CustomOperator(task_id="print-params")
+            runtime_ti = create_runtime_ti(
+                dag_id="c",
+                task=task,
+                conf={
+                    "x": 3,
+                    "text": "Hello World!",
+                    "flag": False,
+                    "a_simple_list": ["one", "two", "three", "actually one value is made per line"],
+                },
+            )
+            run(runtime_ti, context=runtime_ti.get_template_context(), log=mock.MagicMock())
+
+            assert os.getpid() != main_pid
+            os.kill(os.getpid(), signal.SIGTERM)
+            # Ensure that the signal is serviced before we finish and exit the subprocess.
+            sleep(0.5)
+
+        proc = ActivitySubprocess.start(
+            dag_rel_path=os.devnull,
+            bundle_info=FAKE_BUNDLE,
+            what=TaskInstance(
+                id=ti_id,
+                task_id="b",
+                dag_id="c",
+                run_id="d",
+                try_number=1,
+                dag_version_id=uuid7(),
+            ),
+            client=make_client(transport=httpx.MockTransport(handle_request)),
+            target=subprocess_main,
+        )
+
+        proc.wait()
+        captured = capfd.readouterr()
+        assert "On kill hook called!" in captured.out
+
     def test_subprocess_sigkilled(self, client_with_ti_start):
         main_pid = os.getpid()