You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ep...@apache.org on 2023/04/26 14:37:49 UTC
[airflow] branch main updated: Fix KubernetesExecutor sending state to scheduler (#30872)
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 0133f6806d Fix KubernetesExecutor sending state to scheduler (#30872)
0133f6806d is described below
commit 0133f6806dbfb60b84b5bea4ce0daf073c246d52
Author: Jed Cunningham <66...@users.noreply.github.com>
AuthorDate: Wed Apr 26 09:37:39 2023 -0500
Fix KubernetesExecutor sending state to scheduler (#30872)
* Fix KubernetesExecutor sending state to scheduler
It turns out the event_buffer expects the TI state, not the worker
state. This changes KE to sending the proper state back to the scheduler
job.
* dedup using event, not running
---
airflow/executors/kubernetes_executor.py | 41 ++++++++++++++++----
airflow/jobs/scheduler_job_runner.py | 5 ++-
tests/executors/test_kubernetes_executor.py | 58 ++++++++++++++++++++++++++++-
3 files changed, 94 insertions(+), 10 deletions(-)
diff --git a/airflow/executors/kubernetes_executor.py b/airflow/executors/kubernetes_executor.py
index f8e816e9f5..80091aa827 100644
--- a/airflow/executors/kubernetes_executor.py
+++ b/airflow/executors/kubernetes_executor.py
@@ -46,6 +46,7 @@ from airflow.kubernetes.kube_client import get_kube_client
from airflow.kubernetes.kube_config import KubeConfig
from airflow.kubernetes.kubernetes_helper_functions import annotations_to_key, create_pod_id
from airflow.kubernetes.pod_generator import PodGenerator
+from airflow.models.taskinstance import TaskInstance
from airflow.utils.event_scheduler import EventScheduler
from airflow.utils.log.logging_mixin import LoggingMixin
from airflow.utils.session import NEW_SESSION, provide_session
@@ -53,7 +54,7 @@ from airflow.utils.state import State, TaskInstanceState
if TYPE_CHECKING:
from airflow.executors.base_executor import CommandType
- from airflow.models.taskinstance import TaskInstance, TaskInstanceKey
+ from airflow.models.taskinstance import TaskInstanceKey
# TaskInstance key, command, configuration, pod_template_file
KubernetesJobType = Tuple[TaskInstanceKey, CommandType, Any, Optional[str]]
@@ -226,8 +227,24 @@ class KubernetesJobWatcher(multiprocessing.Process, LoggingMixin):
self.log.error("Event: %s Failed", pod_name)
self.watcher_queue.put((pod_name, namespace, State.FAILED, annotations, resource_version))
elif status == "Succeeded":
+ # We get multiple events once the pod hits a terminal state, and we only want to
+ # send it along to the scheduler once.
+ # If our event type is DELETED, we have the POD_EXECUTOR_DONE_KEY, or the pod has
+ # a deletion timestamp, we've already seen the initial Succeeded event and sent it
+ # along to the scheduler.
+ pod = event["object"]
+ if (
+ event["type"] == "DELETED"
+ or POD_EXECUTOR_DONE_KEY in pod.metadata.labels
+ or pod.metadata.deletion_timestamp
+ ):
+ self.log.info(
+ "Skipping event for Succeeded pod %s - event for this pod already sent to executor",
+ pod_name,
+ )
+ return
self.log.info("Event: %s Succeeded", pod_name)
- self.watcher_queue.put((pod_name, namespace, State.SUCCESS, annotations, resource_version))
+ self.watcher_queue.put((pod_name, namespace, None, annotations, resource_version))
elif status == "Running":
if event["type"] == "DELETED":
self.log.info("Event: Pod %s deleted before it could complete", pod_name)
@@ -725,7 +742,15 @@ class KubernetesExecutor(BaseExecutor):
next_event = self.event_scheduler.run(blocking=False)
self.log.debug("Next timed event is in %f", next_event)
- def _change_state(self, key: TaskInstanceKey, state: str | None, pod_name: str, namespace: str) -> None:
+ @provide_session
+ def _change_state(
+ self,
+ key: TaskInstanceKey,
+ state: str | None,
+ pod_name: str,
+ namespace: str,
+ session: Session = NEW_SESSION,
+ ) -> None:
if TYPE_CHECKING:
assert self.kube_scheduler
@@ -745,10 +770,12 @@ class KubernetesExecutor(BaseExecutor):
self.running.remove(key)
except KeyError:
self.log.debug("TI key not in running, not adding to event_buffer: %s", key)
- else:
- # We get multiple events once the pod hits a terminal state, and we only want to
- # do this once, so only do it when we remove the task from running
- self.event_buffer[key] = state, None
+
+ # If we don't have a TI state, look it up from the db. event_buffer expects the TI state
+ if state is None:
+ state = session.query(TaskInstance.state).filter(TaskInstance.filter_for_tis([key])).scalar()
+
+ self.event_buffer[key] = state, None
@staticmethod
def _get_pod_namespace(ti: TaskInstance):
diff --git a/airflow/jobs/scheduler_job_runner.py b/airflow/jobs/scheduler_job_runner.py
index 9d77795c9c..76385a26aa 100644
--- a/airflow/jobs/scheduler_job_runner.py
+++ b/airflow/jobs/scheduler_job_runner.py
@@ -779,7 +779,10 @@ class SchedulerJobRunner(BaseJobRunner, LoggingMixin):
# but that is handled by the zombie detection.
ti_queued = ti.try_number == buffer_key.try_number and ti.state == TaskInstanceState.QUEUED
- ti_requeued = ti.queued_by_job_id != self.job.id or self.job.executor.has_task(ti)
+ ti_requeued = (
+ ti.queued_by_job_id != self.job.id # Another scheduler has queued this task again
+ or self.job.executor.has_task(ti) # This scheduler has this task already
+ )
if ti_queued and not ti_requeued:
Stats.incr(
diff --git a/tests/executors/test_kubernetes_executor.py b/tests/executors/test_kubernetes_executor.py
index 94e092b447..3233b75001 100644
--- a/tests/executors/test_kubernetes_executor.py
+++ b/tests/executors/test_kubernetes_executor.py
@@ -36,10 +36,12 @@ from airflow.models.taskinstance import TaskInstanceKey
from airflow.operators.bash import BashOperator
from airflow.operators.empty import EmptyOperator
from airflow.utils import timezone
+from airflow.utils.state import State, TaskInstanceState
from tests.test_utils.config import conf_vars
try:
from airflow.executors.kubernetes_executor import (
+ POD_EXECUTOR_DONE_KEY,
AirflowKubernetesScheduler,
KubernetesExecutor,
KubernetesJobWatcher,
@@ -50,7 +52,6 @@ try:
from airflow.kubernetes import pod_generator
from airflow.kubernetes.kubernetes_helper_functions import annotations_to_key
from airflow.kubernetes.pod_generator import PodGenerator
- from airflow.utils.state import State
except ImportError:
AirflowKubernetesScheduler = None # type: ignore
@@ -576,6 +577,34 @@ class TestKubernetesExecutor:
finally:
executor.end()
+ @pytest.mark.parametrize(
+ "ti_state", [TaskInstanceState.SUCCESS, TaskInstanceState.FAILED, TaskInstanceState.DEFERRED]
+ )
+ @mock.patch("airflow.executors.kubernetes_executor.KubernetesJobWatcher")
+ @mock.patch("airflow.executors.kubernetes_executor.get_kube_client")
+ @mock.patch("airflow.executors.kubernetes_executor.AirflowKubernetesScheduler.delete_pod")
+ def test_change_state_none(
+ self,
+ mock_delete_pod,
+ mock_get_kube_client,
+ mock_kubernetes_job_watcher,
+ ti_state,
+ create_task_instance,
+ ):
+ """Ensure that when change_state gets state=None, it looks up the TI state from the db"""
+ executor = self.kubernetes_executor
+ executor.start()
+ try:
+ ti = create_task_instance(state=ti_state)
+ key = ti.key
+ executor.running = {key}
+ executor._change_state(key, None, "pod_name", "default")
+ assert executor.event_buffer[key][0] == ti_state
+ assert executor.running == set()
+ mock_delete_pod.assert_called_once_with(pod_name="pod_name", namespace="default")
+ finally:
+ executor.end()
+
@pytest.mark.parametrize(
"multi_namespace_mode_namespace_list, watchers_keys",
[
@@ -1156,6 +1185,7 @@ class TestKubernetesJobWatcher:
annotations={"airflow-worker": "bar", **self.core_annotations},
namespace="airflow",
resource_version="456",
+ labels={},
),
status=k8s.V1PodStatus(phase="Pending"),
)
@@ -1207,7 +1237,31 @@ class TestKubernetesJobWatcher:
self.events.append({"type": "MODIFIED", "object": self.pod})
self._run()
- self.assert_watcher_queue_called_once_with_state(State.SUCCESS)
+ # We don't know the TI state, so we send in None
+ self.assert_watcher_queue_called_once_with_state(None)
+
+ def test_process_status_succeeded_dedup_label(self):
+ self.pod.status.phase = "Succeeded"
+ self.pod.metadata.labels[POD_EXECUTOR_DONE_KEY] = "True"
+ self.events.append({"type": "MODIFIED", "object": self.pod})
+
+ self._run()
+ self.watcher.watcher_queue.put.assert_not_called()
+
+ def test_process_status_succeeded_dedup_timestamp(self):
+ self.pod.status.phase = "Succeeded"
+ self.pod.metadata.deletion_timestamp = datetime.utcnow()
+ self.events.append({"type": "MODIFIED", "object": self.pod})
+
+ self._run()
+ self.watcher.watcher_queue.put.assert_not_called()
+
+ def test_process_status_succeeded_type_delete(self):
+ self.pod.status.phase = "Succeeded"
+ self.events.append({"type": "DELETED", "object": self.pod})
+
+ self._run()
+ self.watcher.watcher_queue.put.assert_not_called()
def test_process_status_running_deleted(self):
self.pod.status.phase = "Running"