[airflow] branch main updated: Work around pyupgrade edge cases (#26384)

[po...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git


The following commit(s) were added to refs/heads/main by this push:
     new 9444d9789b Work around pyupgrade edge cases (#26384)
9444d9789b is described below

commit 9444d9789bc88e1063d81d28e219446b2251c0e1
Author: Tzu-ping Chung <ur...@gmail.com>
AuthorDate: Wed Sep 14 20:25:01 2022 +0800

    Work around pyupgrade edge cases (#26384)
---
 .pre-commit-config.yaml                     |  8 +-------
 airflow/models/baseoperator.py              | 10 +++++++++-
 airflow/providers/google/cloud/hooks/gcs.py | 16 ++++++++++------
 3 files changed, 20 insertions(+), 14 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 60da9658e3..2d329ec4ba 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -213,13 +213,7 @@ repos:
       - id: pyupgrade
         name: Upgrade Python code automatically
         args: ["--py37-plus"]
-        # We need to exclude gcs hook from pyupgrade because it has public "list" command which clashes
-        # with `list` that is used as type
-        # Test Python tests if different kinds of typing including one that does not have
-        # __future__ annotations, so it should be left without automated upgrade
-        # BaseOperator is disabled because replacing ClassVar[List[Type with corresponding list/type causes the attr to fail
-        # see https://github.com/apache/airflow/pull/26290#issuecomment-1246014807
-        exclude: ^airflow/_vendor/|^airflow/providers/google/cloud/hooks/gcs.py$|^tests/decorators/test_python.py$|^airflow/models/baseoperator.py$
+        exclude: ^airflow/_vendor/
   - repo: https://github.com/pre-commit/pygrep-hooks
     rev: v1.9.0
     hooks:
diff --git a/airflow/models/baseoperator.py b/airflow/models/baseoperator.py
index aebbf4fa44..06845910be 100644
--- a/airflow/models/baseoperator.py
+++ b/airflow/models/baseoperator.py
@@ -1738,11 +1738,19 @@ def cross_downstream(
         task.set_downstream(to_tasks)
 
 
+# pyupgrade assumes all type annotations can be lazily evaluated, but this is
+# not the case for attrs-decorated classes, since cattrs needs to evaluate the
+# annotation expressions at runtime, and Python before 3.9.0 does not lazily
+# evaluate those. Putting the expression in a top-level assignment statement
+# communicates this runtime requirement to pyupgrade.
+BaseOperatorClassList = List[Type[BaseOperator]]
+
+
 @attr.s(auto_attribs=True)
 class BaseOperatorLink(metaclass=ABCMeta):
     """Abstract base class that defines how we get an operator link."""
 
-    operators: ClassVar[List[Type[BaseOperator]]] = []
+    operators: ClassVar[BaseOperatorClassList] = []
     """
     This property will be used by Airflow Plugins to find the Operators to which you want
     to assign this Operator Link
diff --git a/airflow/providers/google/cloud/hooks/gcs.py b/airflow/providers/google/cloud/hooks/gcs.py
index 8f27fd7c7d..599bae1455 100644
--- a/airflow/providers/google/cloud/hooks/gcs.py
+++ b/airflow/providers/google/cloud/hooks/gcs.py
@@ -29,7 +29,7 @@ from functools import partial
 from io import BytesIO
 from os import path
 from tempfile import NamedTemporaryFile
-from typing import IO, Callable, Generator, List, Optional, Sequence, Set, TypeVar, cast, overload
+from typing import IO, Callable, Generator, Sequence, TypeVar, cast, overload
 from urllib.parse import urlparse
 
 from google.api_core.exceptions import NotFound
@@ -48,6 +48,10 @@ from airflow.version import version
 RT = TypeVar('RT')
 T = TypeVar("T", bound=Callable)
 
+# GCSHook has a method named 'list' (to junior devs: please don't do this), so
+# we need to create an alias to prevent Mypy being confused.
+List = list
+
 # Use default timeout from google-cloud-storage
 DEFAULT_TIMEOUT = 60
 
@@ -127,7 +131,7 @@ class GCSHook(GoogleBaseHook):
     connection.
     """
 
-    _conn = None  # type: Optional[storage.Client]
+    _conn: storage.Client | None = None
 
     def __init__(
         self,
@@ -671,7 +675,7 @@ class GCSHook(GoogleBaseHook):
         except NotFound:
             self.log.info("Bucket %s not exists", bucket_name)
 
-    def list(self, bucket_name, versions=None, max_results=None, prefix=None, delimiter=None) -> list:
+    def list(self, bucket_name, versions=None, max_results=None, prefix=None, delimiter=None) -> List:
         """
         List all objects from the bucket with the give string prefix in name
 
@@ -1127,11 +1131,11 @@ class GCSHook(GoogleBaseHook):
         # Determine objects to copy and delete
         to_copy = source_names - destination_names
         to_delete = destination_names - source_names
-        to_copy_blobs = {source_names_index[a] for a in to_copy}  # type: Set[storage.Blob]
-        to_delete_blobs = {destination_names_index[a] for a in to_delete}  # type: Set[storage.Blob]
+        to_copy_blobs: set[storage.Blob] = {source_names_index[a] for a in to_copy}
+        to_delete_blobs: set[storage.Blob] = {destination_names_index[a] for a in to_delete}
         # Find names that are in both buckets
         names_to_check = source_names.intersection(destination_names)
-        to_rewrite_blobs = set()  # type: Set[storage.Blob]
+        to_rewrite_blobs: set[storage.Blob] = set()
         # Compare objects based on crc32
         for current_name in names_to_check:
             source_blob = source_names_index[current_name]