[airflow] 16/16: Updates 3.6 limits for latest versions of a few libraries (#15209)

[po...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

potiuk pushed a commit to branch v2-0-test
in repository https://gitbox.apache.org/repos/asf/airflow.git

commit 5f5a914db9352453b2ca11dc415a7837a53be327
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Mon Apr 5 20:25:11 2021 +0200

    Updates 3.6 limits for latest versions of a few libraries (#15209)
    
    This PR sets Pythong 3.6 specific limits for some of the packages
    that recently dropped support for Python 3.6 binary packages
    released via PyPI. Even if those packages did not drop the
    Python 3.6 support entirely, it gets more and more difficult to
    get those packages installed (both locally and in the Docker image)
    because the require the packages to be compiled and they often
    require a number of external dependencies to do so.
    
    This makes it difficult to automatically upgrade dependencies,
    because such upgrade fails for Python 3.6 images if we attempt
    to do so.
    
    This PR limits several of those dependencies (dask/pandas/numpy)
    to not use the lates major releases for those packages but limits
    them to the latest released versions.
    
    Also comment/clarification was added to recently (#15114) added limit
    for `pandas-gbq`. This limit has been added because of broken
    import for bigquery provider, but the comment about it was missing
    so the comment is added now.
    
    (cherry picked from commit e49722859b81cfcdd7e4bb8e8aba4efb049a8590)
---
 airflow/models/baseoperator.py                       |  2 +-
 airflow/providers/google/cloud/operators/dataflow.py |  6 +++---
 setup.cfg                                            |  7 ++++++-
 setup.py                                             | 11 +++++++++--
 4 files changed, 19 insertions(+), 7 deletions(-)

diff --git a/airflow/models/baseoperator.py b/airflow/models/baseoperator.py
index 06094a1..8bda785 100644
--- a/airflow/models/baseoperator.py
+++ b/airflow/models/baseoperator.py
@@ -1493,7 +1493,7 @@ def cross_downstream(
 class BaseOperatorLink(metaclass=ABCMeta):
     """Abstract base class that defines how we get an operator link."""
 
-    operators: ClassVar[List[Type[BaseOperator]]] = []
+    operators: ClassVar[List[Type[BaseOperator]]] = []  # pylint: disable=invalid-name
     """
     This property will be used by Airflow Plugins to find the Operators to which you want
     to assign this Operator Link
diff --git a/airflow/providers/google/cloud/operators/dataflow.py b/airflow/providers/google/cloud/operators/dataflow.py
index 92ae77e..513fea3 100644
--- a/airflow/providers/google/cloud/operators/dataflow.py
+++ b/airflow/providers/google/cloud/operators/dataflow.py
@@ -43,9 +43,9 @@ class CheckJobRunning(Enum):
     WaitForRun - wait for job to finish and then continue with new job
     """
 
-    IgnoreJob = 1
-    FinishIfRunning = 2
-    WaitForRun = 3
+    IgnoreJob = 1  # pylint: disable=invalid-name
+    FinishIfRunning = 2  # pylint: disable=invalid-name
+    WaitForRun = 3  # pylint: disable=invalid-name
 
 
 class DataflowConfiguration:
diff --git a/setup.cfg b/setup.cfg
index 9c051af..1dcdef9 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -110,7 +110,12 @@ install_requires =
     markdown>=2.5.2, <4.0
     markupsafe>=1.1.1, <2.0
     marshmallow-oneofschema>=2.0.1
-    pandas>=0.17.1, <2.0
+    # Numpy stopped releasing 3.6 binaries for 1.20.* series.
+    numpy<1.20;python_version<"3.7"
+    numpy;python_version>="3.7"
+    # Pandas stopped releasing 3.6 binaries for 1.2.* series.
+    pandas>=0.17.1, <1.2;python_version<"3.7"
+    pandas>=0.17.1, <2.0;python_version>="3.7"
     pendulum~=2.0
     pep562~=1.0;python_version<"3.7"
     psutil>=4.2.0, <6.0.0
diff --git a/setup.py b/setup.py
index 0f421bc..51bd9be 100644
--- a/setup.py
+++ b/setup.py
@@ -237,7 +237,12 @@ cgroups = [
 cloudant = [
     'cloudant>=2.0',
 ]
-dask = ['cloudpickle>=1.4.1, <1.5.0', 'distributed>=2.11.1, <2.20']
+dask = [
+    'cloudpickle>=1.4.1, <1.5.0',
+    'dask<2021.3.1;python_version>"3.7"',  # dask stopped supporting python 3.6 in 2021.3.1 version
+    'dask>=2.9.0;python_version>="3.7"',
+    'distributed>=2.11.1, <2.20',
+]
 databricks = [
     'requests>=2.20.0, <3',
 ]
@@ -313,7 +318,9 @@ google = [
     'google-cloud-workflows>=0.1.0,<2.0.0',
     'grpcio-gcp>=0.2.2',
     'json-merge-patch~=0.2',
-    'pandas-gbq',
+    # pandas-gbq 0.15.0 release broke google provider's bigquery import
+    # _check_google_client_version (airflow/providers/google/cloud/hooks/bigquery.py:49)
+    'pandas-gbq<0.15.0',
     'plyvel',
 ]
 grpc = [