You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by jh...@apache.org on 2021/07/08 18:21:52 UTC
[airflow] 02/03: Introduce compat shim airflow.compat.functools
(#15969)
This is an automated email from the ASF dual-hosted git repository.
jhtimmins pushed a commit to branch v2-1-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit e2453602fd9f6055202424a6491c4ef37a85adca
Author: Tzu-ping Chung <tp...@astronomer.io>
AuthorDate: Tue May 25 21:36:00 2021 +0800
Introduce compat shim airflow.compat.functools (#15969)
This module shims 'cached_property' and 'cache' so modules don't need to
all do their own ad-hoc try-except ImportError.
(cherry picked from commit 3db347edcfe444a67e59e8cf0019e80a02dbceab)
---
airflow/cli/commands/connection_command.py | 3 +--
airflow/cli/commands/kubernetes_command.py | 3 +--
airflow/cli/simple_table.py | 2 +-
airflow/compat/__init__.py | 16 +++++++++++++++
airflow/compat/functools.py | 33 ++++++++++++++++++++++++++++++
airflow/configuration.py | 3 +--
airflow/kubernetes/pod_generator.py | 2 +-
airflow/kubernetes/refresh_config.py | 2 +-
airflow/models/baseoperator.py | 6 +-----
airflow/operators/bash.py | 6 +-----
airflow/operators/sql.py | 6 +-----
airflow/providers_manager.py | 2 +-
airflow/secrets/local_filesystem.py | 2 +-
airflow/utils/log/log_reader.py | 6 +-----
airflow/utils/log/secrets_masker.py | 15 +-------------
airflow/utils/yaml.py | 2 +-
airflow/www/views.py | 3 +--
17 files changed, 64 insertions(+), 48 deletions(-)
diff --git a/airflow/cli/commands/connection_command.py b/airflow/cli/commands/connection_command.py
index 19912b6..c79ba6a 100644
--- a/airflow/cli/commands/connection_command.py
+++ b/airflow/cli/commands/connection_command.py
@@ -24,13 +24,12 @@ from urllib.parse import urlparse, urlunparse
from sqlalchemy.orm import exc
-import airflow.utils.yaml as yaml
from airflow.cli.simple_table import AirflowConsole
from airflow.exceptions import AirflowNotFoundException
from airflow.hooks.base import BaseHook
from airflow.models import Connection
from airflow.secrets.local_filesystem import load_connections_dict
-from airflow.utils import cli as cli_utils
+from airflow.utils import cli as cli_utils, yaml
from airflow.utils.cli import suppress_logs_and_warning
from airflow.utils.session import create_session
diff --git a/airflow/cli/commands/kubernetes_command.py b/airflow/cli/commands/kubernetes_command.py
index daf11a3..3aad616 100644
--- a/airflow/cli/commands/kubernetes_command.py
+++ b/airflow/cli/commands/kubernetes_command.py
@@ -22,14 +22,13 @@ from kubernetes import client
from kubernetes.client.api_client import ApiClient
from kubernetes.client.rest import ApiException
-import airflow.utils.yaml as yaml
from airflow.executors.kubernetes_executor import KubeConfig, create_pod_id
from airflow.kubernetes import pod_generator
from airflow.kubernetes.kube_client import get_kube_client
from airflow.kubernetes.pod_generator import PodGenerator
from airflow.models import TaskInstance
from airflow.settings import pod_mutation_hook
-from airflow.utils import cli as cli_utils
+from airflow.utils import cli as cli_utils, yaml
from airflow.utils.cli import get_dag
diff --git a/airflow/cli/simple_table.py b/airflow/cli/simple_table.py
index 65e846e..d17f948 100644
--- a/airflow/cli/simple_table.py
+++ b/airflow/cli/simple_table.py
@@ -24,8 +24,8 @@ from rich.syntax import Syntax
from rich.table import Table
from tabulate import tabulate
-import airflow.utils.yaml as yaml
from airflow.plugins_manager import PluginsDirectorySource
+from airflow.utils import yaml
from airflow.utils.platform import is_tty
diff --git a/airflow/compat/__init__.py b/airflow/compat/__init__.py
new file mode 100644
index 0000000..13a8339
--- /dev/null
+++ b/airflow/compat/__init__.py
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
diff --git a/airflow/compat/functools.py b/airflow/compat/functools.py
new file mode 100644
index 0000000..10b4085
--- /dev/null
+++ b/airflow/compat/functools.py
@@ -0,0 +1,33 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import sys
+
+if sys.version_info >= (3, 8):
+ from functools import cached_property # pylint: disable=no-name-in-module
+else:
+ from cached_property import cached_property
+
+if sys.version_info >= (3, 9):
+ from functools import cache # pylint: disable=no-name-in-module
+else:
+ from functools import lru_cache
+
+ cache = lru_cache(maxsize=None)
+
+
+__all__ = ["cache", "cached_property"]
diff --git a/airflow/configuration.py b/airflow/configuration.py
index 263ba4b..ed38efd 100644
--- a/airflow/configuration.py
+++ b/airflow/configuration.py
@@ -36,6 +36,7 @@ from typing import Dict, List, Optional, Union
from airflow.exceptions import AirflowConfigException
from airflow.secrets import DEFAULT_SECRETS_SEARCH_PATH, BaseSecretsBackend
+from airflow.utils import yaml
from airflow.utils.module_loading import import_string
log = logging.getLogger(__name__)
@@ -97,8 +98,6 @@ def default_config_yaml() -> List[dict]:
:return: Python dictionary containing configs & their info
"""
- import airflow.utils.yaml as yaml
-
with open(_default_config_file_path('config.yml')) as config_file:
return yaml.safe_load(config_file)
diff --git a/airflow/kubernetes/pod_generator.py b/airflow/kubernetes/pod_generator.py
index 80602e3..4b4028f 100644
--- a/airflow/kubernetes/pod_generator.py
+++ b/airflow/kubernetes/pod_generator.py
@@ -34,9 +34,9 @@ from dateutil import parser
from kubernetes.client import models as k8s
from kubernetes.client.api_client import ApiClient
-import airflow.utils.yaml as yaml
from airflow.exceptions import AirflowConfigException
from airflow.kubernetes.pod_generator_deprecated import PodDefaults, PodGenerator as PodGeneratorDeprecated
+from airflow.utils import yaml
from airflow.version import version as airflow_version
MAX_LABEL_LEN = 63
diff --git a/airflow/kubernetes/refresh_config.py b/airflow/kubernetes/refresh_config.py
index 2738b86..a039e7d 100644
--- a/airflow/kubernetes/refresh_config.py
+++ b/airflow/kubernetes/refresh_config.py
@@ -31,7 +31,7 @@ from kubernetes.client import Configuration
from kubernetes.config.exec_provider import ExecProvider
from kubernetes.config.kube_config import KUBE_CONFIG_DEFAULT_LOCATION, KubeConfigLoader
-import airflow.utils.yaml as yaml
+from airflow.utils import yaml
def _parse_timestamp(ts_str: str) -> int:
diff --git a/airflow/models/baseoperator.py b/airflow/models/baseoperator.py
index f74c5f9..7af23d3 100644
--- a/airflow/models/baseoperator.py
+++ b/airflow/models/baseoperator.py
@@ -46,15 +46,11 @@ from typing import (
import attr
import jinja2
-
-try:
- from functools import cached_property
-except ImportError:
- from cached_property import cached_property
from dateutil.relativedelta import relativedelta
from sqlalchemy.orm import Session
import airflow.templates
+from airflow.compat.functools import cached_property
from airflow.configuration import conf
from airflow.exceptions import AirflowException
from airflow.lineage import apply_lineage, prepare_lineage
diff --git a/airflow/operators/bash.py b/airflow/operators/bash.py
index 1cc85e1..66a70a9 100644
--- a/airflow/operators/bash.py
+++ b/airflow/operators/bash.py
@@ -18,11 +18,7 @@
import os
from typing import Dict, Optional
-try:
- from functools import cached_property
-except ImportError:
- from cached_property import cached_property
-
+from airflow.compat.functools import cached_property
from airflow.exceptions import AirflowException, AirflowSkipException
from airflow.hooks.subprocess import SubprocessHook
from airflow.models import BaseOperator
diff --git a/airflow/operators/sql.py b/airflow/operators/sql.py
index 769cf26..8347bcb 100644
--- a/airflow/operators/sql.py
+++ b/airflow/operators/sql.py
@@ -18,11 +18,7 @@
from distutils.util import strtobool
from typing import Any, Dict, Iterable, List, Mapping, Optional, SupportsAbs, Union
-try:
- from functools import cached_property
-except ImportError:
- from cached_property import cached_property
-
+from airflow.compat.functools import cached_property
from airflow.exceptions import AirflowException
from airflow.hooks.base import BaseHook
from airflow.hooks.dbapi import DbApiHook
diff --git a/airflow/providers_manager.py b/airflow/providers_manager.py
index 420135b..9f1469c 100644
--- a/airflow/providers_manager.py
+++ b/airflow/providers_manager.py
@@ -27,7 +27,7 @@ from typing import Any, Dict, NamedTuple, Set
import jsonschema
from wtforms import Field
-import airflow.utils.yaml as yaml
+from airflow.utils import yaml
from airflow.utils.entry_points import entry_points_with_dist
try:
diff --git a/airflow/secrets/local_filesystem.py b/airflow/secrets/local_filesystem.py
index 3ec20e1..d23969f 100644
--- a/airflow/secrets/local_filesystem.py
+++ b/airflow/secrets/local_filesystem.py
@@ -25,7 +25,6 @@ from inspect import signature
from json import JSONDecodeError
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Tuple
-import airflow.utils.yaml as yaml
from airflow.exceptions import (
AirflowException,
AirflowFileParseException,
@@ -33,6 +32,7 @@ from airflow.exceptions import (
FileSyntaxError,
)
from airflow.secrets.base_secrets import BaseSecretsBackend
+from airflow.utils import yaml
from airflow.utils.file import COMMENT_PATTERN
from airflow.utils.log.logging_mixin import LoggingMixin
diff --git a/airflow/utils/log/log_reader.py b/airflow/utils/log/log_reader.py
index f4096eb..0e1f691 100644
--- a/airflow/utils/log/log_reader.py
+++ b/airflow/utils/log/log_reader.py
@@ -18,11 +18,7 @@
import logging
from typing import Dict, Iterator, List, Optional, Tuple
-try:
- from functools import cached_property
-except ImportError:
- from cached_property import cached_property
-
+from airflow.compat.functools import cached_property
from airflow.configuration import conf
from airflow.models import TaskInstance
from airflow.utils.helpers import render_log_filename
diff --git a/airflow/utils/log/secrets_masker.py b/airflow/utils/log/secrets_masker.py
index 1796cbc..8bb7557 100644
--- a/airflow/utils/log/secrets_masker.py
+++ b/airflow/utils/log/secrets_masker.py
@@ -20,20 +20,7 @@ import logging
import re
from typing import TYPE_CHECKING, Iterable, Optional, Set, TypeVar, Union
-try:
- # 3.8+
- from functools import cached_property
-except ImportError:
- from cached_property import cached_property
-
-try:
- # 3.9+
- from functools import cache
-except ImportError:
- from functools import lru_cache
-
- cache = lru_cache(maxsize=None)
-
+from airflow.compat.functools import cache, cached_property
if TYPE_CHECKING:
from airflow.typing_compat import RePatternType
diff --git a/airflow/utils/yaml.py b/airflow/utils/yaml.py
index e3be61c..c452631 100644
--- a/airflow/utils/yaml.py
+++ b/airflow/utils/yaml.py
@@ -30,7 +30,7 @@ import sys
from typing import TYPE_CHECKING, Any, BinaryIO, TextIO, Union, cast
if TYPE_CHECKING:
- from yaml.error import MarkedYAMLError # noqa
+ from yaml.error import MarkedYAMLError, YAMLError # noqa
def safe_load(stream: Union[bytes, str, BinaryIO, TextIO]) -> Any:
diff --git a/airflow/www/views.py b/airflow/www/views.py
index fdfea42..9bc677e 100644
--- a/airflow/www/views.py
+++ b/airflow/www/views.py
@@ -82,7 +82,6 @@ from wtforms import SelectField, validators
from wtforms.validators import InputRequired
import airflow
-import airflow.utils.yaml as yaml
from airflow import models, plugins_manager, settings
from airflow.api.common.experimental.mark_tasks import (
set_dag_run_state_to_failed,
@@ -103,7 +102,7 @@ from airflow.providers_manager import ProvidersManager
from airflow.security import permissions
from airflow.ti_deps.dep_context import DepContext
from airflow.ti_deps.dependencies_deps import RUNNING_DEPS, SCHEDULER_QUEUED_DEPS
-from airflow.utils import json as utils_json, timezone
+from airflow.utils import json as utils_json, timezone, yaml
from airflow.utils.dates import infer_time_unit, scale_time_units
from airflow.utils.docs import get_docs_url
from airflow.utils.helpers import alchemy_to_dict