You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ep...@apache.org on 2023/04/26 14:37:49 UTC

[airflow] branch main updated: Fix KubernetesExecutor sending state to scheduler (#30872)

This is an automated email from the ASF dual-hosted git repository.

ephraimanierobi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git


The following commit(s) were added to refs/heads/main by this push:
     new 0133f6806d Fix KubernetesExecutor sending state to scheduler (#30872)
0133f6806d is described below

commit 0133f6806dbfb60b84b5bea4ce0daf073c246d52
Author: Jed Cunningham <66...@users.noreply.github.com>
AuthorDate: Wed Apr 26 09:37:39 2023 -0500

    Fix KubernetesExecutor sending state to scheduler (#30872)
    
    * Fix KubernetesExecutor sending state to scheduler
    
    It turns out the event_buffer expects the TI state, not the worker
    state. This changes KE to sending the proper state back to the scheduler
    job.
    
    * dedup using event, not running
---
 airflow/executors/kubernetes_executor.py    | 41 ++++++++++++++++----
 airflow/jobs/scheduler_job_runner.py        |  5 ++-
 tests/executors/test_kubernetes_executor.py | 58 ++++++++++++++++++++++++++++-
 3 files changed, 94 insertions(+), 10 deletions(-)

diff --git a/airflow/executors/kubernetes_executor.py b/airflow/executors/kubernetes_executor.py
index f8e816e9f5..80091aa827 100644
--- a/airflow/executors/kubernetes_executor.py
+++ b/airflow/executors/kubernetes_executor.py
@@ -46,6 +46,7 @@ from airflow.kubernetes.kube_client import get_kube_client
 from airflow.kubernetes.kube_config import KubeConfig
 from airflow.kubernetes.kubernetes_helper_functions import annotations_to_key, create_pod_id
 from airflow.kubernetes.pod_generator import PodGenerator
+from airflow.models.taskinstance import TaskInstance
 from airflow.utils.event_scheduler import EventScheduler
 from airflow.utils.log.logging_mixin import LoggingMixin
 from airflow.utils.session import NEW_SESSION, provide_session
@@ -53,7 +54,7 @@ from airflow.utils.state import State, TaskInstanceState
 
 if TYPE_CHECKING:
     from airflow.executors.base_executor import CommandType
-    from airflow.models.taskinstance import TaskInstance, TaskInstanceKey
+    from airflow.models.taskinstance import TaskInstanceKey
 
     # TaskInstance key, command, configuration, pod_template_file
     KubernetesJobType = Tuple[TaskInstanceKey, CommandType, Any, Optional[str]]
@@ -226,8 +227,24 @@ class KubernetesJobWatcher(multiprocessing.Process, LoggingMixin):
             self.log.error("Event: %s Failed", pod_name)
             self.watcher_queue.put((pod_name, namespace, State.FAILED, annotations, resource_version))
         elif status == "Succeeded":
+            # We get multiple events once the pod hits a terminal state, and we only want to
+            # send it along to the scheduler once.
+            # If our event type is DELETED, we have the POD_EXECUTOR_DONE_KEY, or the pod has
+            # a deletion timestamp, we've already seen the initial Succeeded event and sent it
+            # along to the scheduler.
+            pod = event["object"]
+            if (
+                event["type"] == "DELETED"
+                or POD_EXECUTOR_DONE_KEY in pod.metadata.labels
+                or pod.metadata.deletion_timestamp
+            ):
+                self.log.info(
+                    "Skipping event for Succeeded pod %s - event for this pod already sent to executor",
+                    pod_name,
+                )
+                return
             self.log.info("Event: %s Succeeded", pod_name)
-            self.watcher_queue.put((pod_name, namespace, State.SUCCESS, annotations, resource_version))
+            self.watcher_queue.put((pod_name, namespace, None, annotations, resource_version))
         elif status == "Running":
             if event["type"] == "DELETED":
                 self.log.info("Event: Pod %s deleted before it could complete", pod_name)
@@ -725,7 +742,15 @@ class KubernetesExecutor(BaseExecutor):
         next_event = self.event_scheduler.run(blocking=False)
         self.log.debug("Next timed event is in %f", next_event)
 
-    def _change_state(self, key: TaskInstanceKey, state: str | None, pod_name: str, namespace: str) -> None:
+    @provide_session
+    def _change_state(
+        self,
+        key: TaskInstanceKey,
+        state: str | None,
+        pod_name: str,
+        namespace: str,
+        session: Session = NEW_SESSION,
+    ) -> None:
         if TYPE_CHECKING:
             assert self.kube_scheduler
 
@@ -745,10 +770,12 @@ class KubernetesExecutor(BaseExecutor):
             self.running.remove(key)
         except KeyError:
             self.log.debug("TI key not in running, not adding to event_buffer: %s", key)
-        else:
-            # We get multiple events once the pod hits a terminal state, and we only want to
-            # do this once, so only do it when we remove the task from running
-            self.event_buffer[key] = state, None
+
+        # If we don't have a TI state, look it up from the db. event_buffer expects the TI state
+        if state is None:
+            state = session.query(TaskInstance.state).filter(TaskInstance.filter_for_tis([key])).scalar()
+
+        self.event_buffer[key] = state, None
 
     @staticmethod
     def _get_pod_namespace(ti: TaskInstance):
diff --git a/airflow/jobs/scheduler_job_runner.py b/airflow/jobs/scheduler_job_runner.py
index 9d77795c9c..76385a26aa 100644
--- a/airflow/jobs/scheduler_job_runner.py
+++ b/airflow/jobs/scheduler_job_runner.py
@@ -779,7 +779,10 @@ class SchedulerJobRunner(BaseJobRunner, LoggingMixin):
             # but that is handled by the zombie detection.
 
             ti_queued = ti.try_number == buffer_key.try_number and ti.state == TaskInstanceState.QUEUED
-            ti_requeued = ti.queued_by_job_id != self.job.id or self.job.executor.has_task(ti)
+            ti_requeued = (
+                ti.queued_by_job_id != self.job.id  # Another scheduler has queued this task again
+                or self.job.executor.has_task(ti)  # This scheduler has this task already
+            )
 
             if ti_queued and not ti_requeued:
                 Stats.incr(
diff --git a/tests/executors/test_kubernetes_executor.py b/tests/executors/test_kubernetes_executor.py
index 94e092b447..3233b75001 100644
--- a/tests/executors/test_kubernetes_executor.py
+++ b/tests/executors/test_kubernetes_executor.py
@@ -36,10 +36,12 @@ from airflow.models.taskinstance import TaskInstanceKey
 from airflow.operators.bash import BashOperator
 from airflow.operators.empty import EmptyOperator
 from airflow.utils import timezone
+from airflow.utils.state import State, TaskInstanceState
 from tests.test_utils.config import conf_vars
 
 try:
     from airflow.executors.kubernetes_executor import (
+        POD_EXECUTOR_DONE_KEY,
         AirflowKubernetesScheduler,
         KubernetesExecutor,
         KubernetesJobWatcher,
@@ -50,7 +52,6 @@ try:
     from airflow.kubernetes import pod_generator
     from airflow.kubernetes.kubernetes_helper_functions import annotations_to_key
     from airflow.kubernetes.pod_generator import PodGenerator
-    from airflow.utils.state import State
 except ImportError:
     AirflowKubernetesScheduler = None  # type: ignore
 
@@ -576,6 +577,34 @@ class TestKubernetesExecutor:
         finally:
             executor.end()
 
+    @pytest.mark.parametrize(
+        "ti_state", [TaskInstanceState.SUCCESS, TaskInstanceState.FAILED, TaskInstanceState.DEFERRED]
+    )
+    @mock.patch("airflow.executors.kubernetes_executor.KubernetesJobWatcher")
+    @mock.patch("airflow.executors.kubernetes_executor.get_kube_client")
+    @mock.patch("airflow.executors.kubernetes_executor.AirflowKubernetesScheduler.delete_pod")
+    def test_change_state_none(
+        self,
+        mock_delete_pod,
+        mock_get_kube_client,
+        mock_kubernetes_job_watcher,
+        ti_state,
+        create_task_instance,
+    ):
+        """Ensure that when change_state gets state=None, it looks up the TI state from the db"""
+        executor = self.kubernetes_executor
+        executor.start()
+        try:
+            ti = create_task_instance(state=ti_state)
+            key = ti.key
+            executor.running = {key}
+            executor._change_state(key, None, "pod_name", "default")
+            assert executor.event_buffer[key][0] == ti_state
+            assert executor.running == set()
+            mock_delete_pod.assert_called_once_with(pod_name="pod_name", namespace="default")
+        finally:
+            executor.end()
+
     @pytest.mark.parametrize(
         "multi_namespace_mode_namespace_list, watchers_keys",
         [
@@ -1156,6 +1185,7 @@ class TestKubernetesJobWatcher:
                 annotations={"airflow-worker": "bar", **self.core_annotations},
                 namespace="airflow",
                 resource_version="456",
+                labels={},
             ),
             status=k8s.V1PodStatus(phase="Pending"),
         )
@@ -1207,7 +1237,31 @@ class TestKubernetesJobWatcher:
         self.events.append({"type": "MODIFIED", "object": self.pod})
 
         self._run()
-        self.assert_watcher_queue_called_once_with_state(State.SUCCESS)
+        # We don't know the TI state, so we send in None
+        self.assert_watcher_queue_called_once_with_state(None)
+
+    def test_process_status_succeeded_dedup_label(self):
+        self.pod.status.phase = "Succeeded"
+        self.pod.metadata.labels[POD_EXECUTOR_DONE_KEY] = "True"
+        self.events.append({"type": "MODIFIED", "object": self.pod})
+
+        self._run()
+        self.watcher.watcher_queue.put.assert_not_called()
+
+    def test_process_status_succeeded_dedup_timestamp(self):
+        self.pod.status.phase = "Succeeded"
+        self.pod.metadata.deletion_timestamp = datetime.utcnow()
+        self.events.append({"type": "MODIFIED", "object": self.pod})
+
+        self._run()
+        self.watcher.watcher_queue.put.assert_not_called()
+
+    def test_process_status_succeeded_type_delete(self):
+        self.pod.status.phase = "Succeeded"
+        self.events.append({"type": "DELETED", "object": self.pod})
+
+        self._run()
+        self.watcher.watcher_queue.put.assert_not_called()
 
     def test_process_status_running_deleted(self):
         self.pod.status.phase = "Running"