You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ep...@apache.org on 2023/10/29 22:39:54 UTC
(airflow) 02/44: Improve modules import in Airflow core by some of them into a type-checking block (#33755)
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch v2-7-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 265126de56708ee74236e1c60a326b8694be52a0
Author: Hussein Awala <hu...@awala.fr>
AuthorDate: Mon Aug 28 16:54:08 2023 +0200
Improve modules import in Airflow core by some of them into a type-checking block (#33755)
* Improve modules import in Airflow core by some of them into a type-checking block
* Restore lazy import in models and fix import in utils.sessions
* fix unit tests
* fix unit tests
* fix static checks
(cherry picked from commit b82ce61285f3f4f0c7eccb2c3effaef53c9fb84e)
---
airflow/configuration.py | 9 ++++++---
airflow/exceptions.py | 3 ++-
airflow/executors/base_executor.py | 8 +++++---
airflow/executors/local_executor.py | 6 ++++--
airflow/kubernetes/pre_7_4_0_compatibility/k8s_model.py | 4 +++-
airflow/kubernetes/pre_7_4_0_compatibility/pod_generator.py | 5 ++++-
airflow/plugins_manager.py | 13 ++++++-------
airflow/providers_manager.py | 2 ++
airflow/settings.py | 6 ++++--
airflow/stats.py | 5 ++++-
airflow/templates.py | 5 ++++-
airflow/timetables/_cron.py | 6 ++++--
airflow/timetables/base.py | 4 ++--
airflow/timetables/events.py | 10 +++++++---
airflow/timetables/interval.py | 7 +++++--
airflow/timetables/simple.py | 3 ++-
airflow/timetables/trigger.py | 12 ++++++++----
airflow/triggers/external_task.py | 11 ++++++++---
18 files changed, 80 insertions(+), 39 deletions(-)
diff --git a/airflow/configuration.py b/airflow/configuration.py
index fb3af35852..007f9cc5c4 100644
--- a/airflow/configuration.py
+++ b/airflow/configuration.py
@@ -36,22 +36,25 @@ from configparser import ConfigParser, NoOptionError, NoSectionError
from contextlib import contextmanager
from copy import deepcopy
from json.decoder import JSONDecodeError
-from typing import IO, Any, Dict, Generator, Iterable, Pattern, Set, Tuple, Union
+from typing import IO, TYPE_CHECKING, Any, Dict, Generator, Iterable, Pattern, Set, Tuple, Union
from urllib.parse import urlsplit
import re2
from packaging.version import parse as parse_version
from typing_extensions import overload
-from airflow.auth.managers.base_auth_manager import BaseAuthManager
from airflow.exceptions import AirflowConfigException
-from airflow.secrets import DEFAULT_SECRETS_SEARCH_PATH, BaseSecretsBackend
+from airflow.secrets import DEFAULT_SECRETS_SEARCH_PATH
from airflow.utils import yaml
from airflow.utils.empty_set import _get_empty_set_for_configuration
from airflow.utils.module_loading import import_string
from airflow.utils.providers_configuration_loader import providers_configuration_loaded
from airflow.utils.weight_rule import WeightRule
+if TYPE_CHECKING:
+ from airflow.auth.managers.base_auth_manager import BaseAuthManager
+ from airflow.secrets import BaseSecretsBackend
+
log = logging.getLogger(__name__)
# show Airflow's deprecation warnings
diff --git a/airflow/exceptions.py b/airflow/exceptions.py
index b471297cd9..0840e801a1 100644
--- a/airflow/exceptions.py
+++ b/airflow/exceptions.py
@@ -20,7 +20,6 @@
"""Exceptions used by Airflow."""
from __future__ import annotations
-import datetime
import warnings
from http import HTTPStatus
from typing import TYPE_CHECKING, Any, NamedTuple, Sized
@@ -28,6 +27,8 @@ from typing import TYPE_CHECKING, Any, NamedTuple, Sized
from airflow.utils.trigger_rule import TriggerRule
if TYPE_CHECKING:
+ import datetime
+
from airflow.models import DAG, DagRun
diff --git a/airflow/executors/base_executor.py b/airflow/executors/base_executor.py
index b9c2991d24..d364d2315b 100644
--- a/airflow/executors/base_executor.py
+++ b/airflow/executors/base_executor.py
@@ -17,18 +17,16 @@
"""Base executor - this is the base class for all the implemented executors."""
from __future__ import annotations
-import argparse
import logging
import sys
import warnings
from collections import defaultdict
from dataclasses import dataclass, field
-from datetime import datetime
from typing import TYPE_CHECKING, Any, List, Optional, Sequence, Tuple
import pendulum
-from airflow.cli.cli_config import DefaultHelpParser, GroupCommand
+from airflow.cli.cli_config import DefaultHelpParser
from airflow.configuration import conf
from airflow.exceptions import RemovedInAirflow3Warning
from airflow.stats import Stats
@@ -38,8 +36,12 @@ from airflow.utils.state import TaskInstanceState
PARALLELISM: int = conf.getint("core", "PARALLELISM")
if TYPE_CHECKING:
+ import argparse
+ from datetime import datetime
+
from airflow.callbacks.base_callback_sink import BaseCallbackSink
from airflow.callbacks.callback_requests import CallbackRequest
+ from airflow.cli.cli_config import GroupCommand
from airflow.models.taskinstance import TaskInstance
from airflow.models.taskinstancekey import TaskInstanceKey
diff --git a/airflow/executors/local_executor.py b/airflow/executors/local_executor.py
index 8621aa9b13..16aa649ee7 100644
--- a/airflow/executors/local_executor.py
+++ b/airflow/executors/local_executor.py
@@ -30,8 +30,7 @@ import os
import subprocess
from abc import abstractmethod
from multiprocessing import Manager, Process
-from multiprocessing.managers import SyncManager
-from queue import Empty, Queue
+from queue import Empty
from typing import TYPE_CHECKING, Any, Optional, Tuple
from setproctitle import getproctitle, setproctitle
@@ -43,6 +42,9 @@ from airflow.utils.log.logging_mixin import LoggingMixin
from airflow.utils.state import TaskInstanceState
if TYPE_CHECKING:
+ from multiprocessing.managers import SyncManager
+ from queue import Queue
+
from airflow.executors.base_executor import CommandType
from airflow.models.taskinstance import TaskInstanceStateType
from airflow.models.taskinstancekey import TaskInstanceKey
diff --git a/airflow/kubernetes/pre_7_4_0_compatibility/k8s_model.py b/airflow/kubernetes/pre_7_4_0_compatibility/k8s_model.py
index 8280a3265f..cff12d057b 100644
--- a/airflow/kubernetes/pre_7_4_0_compatibility/k8s_model.py
+++ b/airflow/kubernetes/pre_7_4_0_compatibility/k8s_model.py
@@ -19,8 +19,10 @@ from __future__ import annotations
from abc import ABC, abstractmethod
from functools import reduce
+from typing import TYPE_CHECKING
-from kubernetes.client import models as k8s
+if TYPE_CHECKING:
+ from kubernetes.client import models as k8s
class K8SModel(ABC):
diff --git a/airflow/kubernetes/pre_7_4_0_compatibility/pod_generator.py b/airflow/kubernetes/pre_7_4_0_compatibility/pod_generator.py
index 021dd36687..73671fd25a 100644
--- a/airflow/kubernetes/pre_7_4_0_compatibility/pod_generator.py
+++ b/airflow/kubernetes/pre_7_4_0_compatibility/pod_generator.py
@@ -27,13 +27,13 @@ is supported and no serialization need be written.
from __future__ import annotations
import copy
-import datetime
import logging
import os
import secrets
import string
import warnings
from functools import reduce
+from typing import TYPE_CHECKING
import re2
from dateutil import parser
@@ -54,6 +54,9 @@ from airflow.utils import yaml
from airflow.utils.hashlib_wrapper import md5
from airflow.version import version as airflow_version
+if TYPE_CHECKING:
+ import datetime
+
log = logging.getLogger(__name__)
MAX_LABEL_LEN = 63
diff --git a/airflow/plugins_manager.py b/airflow/plugins_manager.py
index ef51175b53..beab0f3b07 100644
--- a/airflow/plugins_manager.py
+++ b/airflow/plugins_manager.py
@@ -29,19 +29,18 @@ import types
from pathlib import Path
from typing import TYPE_CHECKING, Any, Iterable
-try:
- import importlib_metadata
-except ImportError:
- from importlib import metadata as importlib_metadata # type: ignore[no-redef]
-
-from types import ModuleType
-
from airflow import settings
from airflow.utils.entry_points import entry_points_with_dist
from airflow.utils.file import find_path_from_directory
from airflow.utils.module_loading import import_string, qualname
if TYPE_CHECKING:
+ try:
+ import importlib_metadata
+ except ImportError:
+ from importlib import metadata as importlib_metadata # type: ignore[no-redef]
+ from types import ModuleType
+
from airflow.hooks.base import BaseHook
from airflow.listeners.listener import ListenerManager
from airflow.timetables.base import Timetable
diff --git a/airflow/providers_manager.py b/airflow/providers_manager.py
index 9799ca82f6..ca5778bbc3 100644
--- a/airflow/providers_manager.py
+++ b/airflow/providers_manager.py
@@ -83,6 +83,8 @@ def _ensure_prefix_for_placeholders(field_behaviors: dict[str, Any], conn_type:
if TYPE_CHECKING:
+ from typing_extensions import Literal
+
from airflow.decorators.base import TaskDecorator
from airflow.hooks.base import BaseHook
diff --git a/airflow/settings.py b/airflow/settings.py
index 941b58e5b8..a278316d5a 100644
--- a/airflow/settings.py
+++ b/airflow/settings.py
@@ -30,8 +30,7 @@ import pendulum
import pluggy
import sqlalchemy
from sqlalchemy import create_engine, exc, text
-from sqlalchemy.engine import Engine
-from sqlalchemy.orm import Session as SASession, scoped_session, sessionmaker
+from sqlalchemy.orm import scoped_session, sessionmaker
from sqlalchemy.pool import NullPool
from airflow import policies
@@ -43,6 +42,9 @@ from airflow.utils.orm_event_handlers import setup_event_handlers
from airflow.utils.state import State
if TYPE_CHECKING:
+ from sqlalchemy.engine import Engine
+ from sqlalchemy.orm import Session as SASession
+
from airflow.www.utils import UIAlert
log = logging.getLogger(__name__)
diff --git a/airflow/stats.py b/airflow/stats.py
index 93c70deab5..569bce4806 100644
--- a/airflow/stats.py
+++ b/airflow/stats.py
@@ -22,7 +22,10 @@ import socket
from typing import TYPE_CHECKING, Callable
from airflow.configuration import conf
-from airflow.metrics.base_stats_logger import NoStatsLogger, StatsLogger
+from airflow.metrics.base_stats_logger import NoStatsLogger
+
+if TYPE_CHECKING:
+ from airflow.metrics.base_stats_logger import StatsLogger
log = logging.getLogger(__name__)
diff --git a/airflow/templates.py b/airflow/templates.py
index 8cd113054d..95851253a7 100644
--- a/airflow/templates.py
+++ b/airflow/templates.py
@@ -17,11 +17,14 @@
# under the License.
from __future__ import annotations
-import datetime
+from typing import TYPE_CHECKING
import jinja2.nativetypes
import jinja2.sandbox
+if TYPE_CHECKING:
+ import datetime
+
class _AirflowEnvironmentMixin:
def __init__(self, **kwargs):
diff --git a/airflow/timetables/_cron.py b/airflow/timetables/_cron.py
index 89cae4bdcb..6787628888 100644
--- a/airflow/timetables/_cron.py
+++ b/airflow/timetables/_cron.py
@@ -18,17 +18,19 @@ from __future__ import annotations
import datetime
from functools import cached_property
-from typing import Any
+from typing import TYPE_CHECKING, Any
from cron_descriptor import CasingTypeEnum, ExpressionDescriptor, FormatException, MissingFieldException
from croniter import CroniterBadCronError, CroniterBadDateError, croniter
-from pendulum import DateTime
from pendulum.tz.timezone import Timezone
from airflow.exceptions import AirflowTimetableInvalid
from airflow.utils.dates import cron_presets
from airflow.utils.timezone import convert_to_utc, make_aware, make_naive
+if TYPE_CHECKING:
+ from pendulum import DateTime
+
def _is_schedule_fixed(expression: str) -> bool:
"""Figures out if the schedule has a fixed time (e.g. 3 AM every day).
diff --git a/airflow/timetables/base.py b/airflow/timetables/base.py
index c02f700233..b5e95ef5f4 100644
--- a/airflow/timetables/base.py
+++ b/airflow/timetables/base.py
@@ -19,11 +19,11 @@ from __future__ import annotations
from typing import TYPE_CHECKING, Any, NamedTuple, Sequence
from warnings import warn
-from pendulum import DateTime
-
from airflow.typing_compat import Protocol, runtime_checkable
if TYPE_CHECKING:
+ from pendulum import DateTime
+
from airflow.utils.types import DagRunType
diff --git a/airflow/timetables/events.py b/airflow/timetables/events.py
index ce8fa9527f..62cf3dce4e 100644
--- a/airflow/timetables/events.py
+++ b/airflow/timetables/events.py
@@ -17,12 +17,16 @@
from __future__ import annotations
import itertools
-from typing import Iterable
+from typing import TYPE_CHECKING, Iterable
import pendulum
-from pendulum import DateTime
-from airflow.timetables.base import DagRunInfo, DataInterval, TimeRestriction, Timetable
+from airflow.timetables.base import DagRunInfo, DataInterval, Timetable
+
+if TYPE_CHECKING:
+ from pendulum import DateTime
+
+ from airflow.timetables.base import TimeRestriction
class EventsTimetable(Timetable):
diff --git a/airflow/timetables/interval.py b/airflow/timetables/interval.py
index 27e128ff52..077c4195a7 100644
--- a/airflow/timetables/interval.py
+++ b/airflow/timetables/interval.py
@@ -17,16 +17,19 @@
from __future__ import annotations
import datetime
-from typing import Any, Union
+from typing import TYPE_CHECKING, Any, Union
from dateutil.relativedelta import relativedelta
from pendulum import DateTime
from airflow.exceptions import AirflowTimetableInvalid
from airflow.timetables._cron import CronMixin
-from airflow.timetables.base import DagRunInfo, DataInterval, TimeRestriction, Timetable
+from airflow.timetables.base import DagRunInfo, DataInterval, Timetable
from airflow.utils.timezone import convert_to_utc
+if TYPE_CHECKING:
+ from airflow.timetables.base import TimeRestriction
+
Delta = Union[datetime.timedelta, relativedelta]
diff --git a/airflow/timetables/simple.py b/airflow/timetables/simple.py
index 53ddf6a7a8..0dd73627d6 100644
--- a/airflow/timetables/simple.py
+++ b/airflow/timetables/simple.py
@@ -21,12 +21,13 @@ from typing import TYPE_CHECKING, Any, Collection
from pendulum import DateTime
-from airflow.timetables.base import DagRunInfo, DataInterval, TimeRestriction, Timetable
+from airflow.timetables.base import DagRunInfo, DataInterval, Timetable
if TYPE_CHECKING:
from sqlalchemy import Session
from airflow.models.dataset import DatasetEvent
+ from airflow.timetables.base import TimeRestriction
from airflow.utils.types import DagRunType
diff --git a/airflow/timetables/trigger.py b/airflow/timetables/trigger.py
index e5c5b7c153..95d2923803 100644
--- a/airflow/timetables/trigger.py
+++ b/airflow/timetables/trigger.py
@@ -17,14 +17,18 @@
from __future__ import annotations
import datetime
-from typing import Any
+from typing import TYPE_CHECKING, Any
-from dateutil.relativedelta import relativedelta
from pendulum import DateTime
-from pendulum.tz.timezone import Timezone
from airflow.timetables._cron import CronMixin
-from airflow.timetables.base import DagRunInfo, DataInterval, TimeRestriction, Timetable
+from airflow.timetables.base import DagRunInfo, DataInterval, Timetable
+
+if TYPE_CHECKING:
+ from dateutil.relativedelta import relativedelta
+ from pendulum.tz.timezone import Timezone
+
+ from airflow.timetables.base import TimeRestriction
class CronTriggerTimetable(CronMixin, Timetable):
diff --git a/airflow/triggers/external_task.py b/airflow/triggers/external_task.py
index 2a205d1f91..c353c01159 100644
--- a/airflow/triggers/external_task.py
+++ b/airflow/triggers/external_task.py
@@ -18,18 +18,23 @@ from __future__ import annotations
import asyncio
import typing
-from datetime import datetime
from asgiref.sync import sync_to_async
from sqlalchemy import func
-from sqlalchemy.orm import Session
from airflow.models import DagRun, TaskInstance
from airflow.triggers.base import BaseTrigger, TriggerEvent
from airflow.utils.session import NEW_SESSION, provide_session
-from airflow.utils.state import DagRunState, TaskInstanceState
+from airflow.utils.state import TaskInstanceState
from airflow.utils.timezone import utcnow
+if typing.TYPE_CHECKING:
+ from datetime import datetime
+
+ from sqlalchemy.orm import Session
+
+ from airflow.utils.state import DagRunState
+
class TaskStateTrigger(BaseTrigger):
"""