You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2023/08/08 20:46:31 UTC
[airflow] branch main updated: Refactor: Simplify code in providers/cncf (#33230)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 056f1670fe Refactor: Simplify code in providers/cncf (#33230)
056f1670fe is described below
commit 056f1670fe85a6f9d70cd72ec2e88287e20e6827
Author: Miroslav Šedivý <67...@users.noreply.github.com>
AuthorDate: Tue Aug 8 20:46:23 2023 +0000
Refactor: Simplify code in providers/cncf (#33230)
---
.../cncf/kubernetes/executors/kubernetes_executor.py | 2 +-
airflow/providers/cncf/kubernetes/pod_generator.py | 2 +-
airflow/providers/cncf/kubernetes/triggers/pod.py | 2 +-
airflow/providers/cncf/kubernetes/utils/pod_manager.py | 14 ++++++--------
4 files changed, 9 insertions(+), 11 deletions(-)
diff --git a/airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py b/airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py
index 5e6679556a..a364bf677c 100644
--- a/airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py
+++ b/airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py
@@ -371,7 +371,7 @@ class KubernetesExecutor(BaseExecutor):
from airflow.providers.cncf.kubernetes.executors.kubernetes_executor_utils import ResourceVersion
resource_instance = ResourceVersion()
- for ns in resource_instance.resource_version.keys():
+ for ns in resource_instance.resource_version:
resource_instance.resource_version[ns] = (
last_resource_version[ns] or resource_instance.resource_version[ns]
)
diff --git a/airflow/providers/cncf/kubernetes/pod_generator.py b/airflow/providers/cncf/kubernetes/pod_generator.py
index c0f2e7e89f..39f58b9aa7 100644
--- a/airflow/providers/cncf/kubernetes/pod_generator.py
+++ b/airflow/providers/cncf/kubernetes/pod_generator.py
@@ -606,7 +606,7 @@ def merge_objects(base_obj, client_obj):
base_obj_cp.update(client_obj_cp)
return base_obj_cp
- for base_key in base_obj.to_dict().keys():
+ for base_key in base_obj.to_dict():
base_val = getattr(base_obj, base_key, None)
if not getattr(client_obj, base_key, None) and base_val:
if not isinstance(client_obj_cp, dict):
diff --git a/airflow/providers/cncf/kubernetes/triggers/pod.py b/airflow/providers/cncf/kubernetes/triggers/pod.py
index 6443dfb63f..8c8820cd45 100644
--- a/airflow/providers/cncf/kubernetes/triggers/pod.py
+++ b/airflow/providers/cncf/kubernetes/triggers/pod.py
@@ -246,7 +246,7 @@ class KubernetesPodTrigger(BaseTrigger):
if pod_containers is None:
return ContainerState.UNDEFINED
- container = [c for c in pod_containers if c.name == self.base_container_name][0]
+ container = next(c for c in pod_containers if c.name == self.base_container_name)
for state in (ContainerState.RUNNING, ContainerState.WAITING, ContainerState.TERMINATED):
state_obj = getattr(container.state, state)
diff --git a/airflow/providers/cncf/kubernetes/utils/pod_manager.py b/airflow/providers/cncf/kubernetes/utils/pod_manager.py
index 81b6c1b2ca..aff81a1f18 100644
--- a/airflow/providers/cncf/kubernetes/utils/pod_manager.py
+++ b/airflow/providers/cncf/kubernetes/utils/pod_manager.py
@@ -18,6 +18,7 @@
from __future__ import annotations
import enum
+import itertools as it
import json
import logging
import math
@@ -26,7 +27,7 @@ import warnings
from collections.abc import Iterable
from contextlib import closing, suppress
from dataclasses import dataclass
-from datetime import datetime, timedelta
+from datetime import timedelta
from typing import TYPE_CHECKING, Generator, cast
import pendulum
@@ -324,14 +325,13 @@ class PodManager(LoggingMixin):
(if pod is pending for too long, fails task)
:return:
"""
- curr_time = datetime.now()
+ curr_time = time.time()
while True:
remote_pod = self.read_pod(pod)
if remote_pod.status.phase != PodPhase.PENDING:
break
self.log.warning("Pod not yet started: %s", pod.metadata.name)
- delta = datetime.now() - curr_time
- if delta.total_seconds() >= startup_timeout:
+ if time.time() - curr_time >= startup_timeout:
msg = (
f"Pod took longer than {startup_timeout} seconds to start. "
"Check the pod events in kubernetes to determine why."
@@ -628,14 +628,12 @@ class PodManager(LoggingMixin):
def await_xcom_sidecar_container_start(self, pod: V1Pod) -> None:
self.log.info("Checking if xcom sidecar container is started.")
- warned = False
- while True:
+ for attempt in it.count():
if self.container_is_running(pod, PodDefaults.SIDECAR_CONTAINER_NAME):
self.log.info("The xcom sidecar container is started.")
break
- if not warned:
+ if not attempt:
self.log.warning("The xcom sidecar container is not yet started.")
- warned = True
time.sleep(1)
def extract_xcom(self, pod: V1Pod) -> str: