You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ep...@apache.org on 2023/09/01 15:24:14 UTC
[airflow] 12/16: Reorganize devel_only extra in airflow's setup.py (#33907)
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch v2-7-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 4dd75306e39f0aedbf5c825c8300b65fe4ed8913
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Thu Aug 31 07:35:28 2023 +0200
Reorganize devel_only extra in airflow's setup.py (#33907)
The `devel_only` extra puts together all dependencies that are
needed for CI image and in order to run tests in local virtualenv
for various test cases of ours - but they are not needed as dependencies
of particular providers. They were a little "bag of everything"
and they were hiding some unused dependencies or dependencies that
were either unused or they were actually provider dependencies already.
For example we had qds-sdk dependency there which was really the
qubole provider dependency and it held us back from removing
deprecated boto library from CI image (removed in #33889).
This PR organizes the dependency a bit better:
* split it to logical groups
* removes some unused dependencies
* moves "amazon" mypy dependency from providers to here
At later stage we will move the provider ones into "[devel]" extras
of the providers as part of provider decooupling, but this
will require a bit more changes in CI image building and some
documentation update for developers. This is an intermediate step
to organize it better.
(cherry picked from commit b497234eda4b4bdfda0b09e3674012c6ebebaf2b)
---
airflow/providers/amazon/provider.yaml | 8 ++-
generated/provider_dependencies.json | 4 --
setup.py | 93 +++++++++++++++++++++++-----------
3 files changed, 67 insertions(+), 38 deletions(-)
diff --git a/airflow/providers/amazon/provider.yaml b/airflow/providers/amazon/provider.yaml
index ea7edd1304..2670e299e7 100644
--- a/airflow/providers/amazon/provider.yaml
+++ b/airflow/providers/amazon/provider.yaml
@@ -73,12 +73,10 @@ dependencies:
# set it to the version that `aiobotocore` supports (see `aiobotocore` optional dependency at the end
# of this file). Currently we set aiobotocore as minimum 2.5.3 - as this is was the first version
# that supported boto3 1.28. NOTE!!! BOTOCORE VERSIONS ARE SHIFTED BY 3 MINOR VERSIONS
+ # NOTE!!! Make sure to update _MIN_BOTO3_VERSION in setup.py when you update it here
- boto3>=1.28.0
- - mypy-boto3-rds>=1.28.0
- - mypy-boto3-redshift-data>=1.28.0
- - mypy-boto3-s3>=1.28.0
- - mypy-boto3-appflow>=1.28.0
- # NOTE!!! BOTOCORE VERSIONS ARE SHIFTED BY 3 MINOR VERSIONS
+ # NOTE!!! BOTOCORE version is always shifted by 3 MINOR VERSIONS from boto3
+ # See https://github.com/boto/boto3/issues/2702
- botocore>=1.31.0
- asgiref
# watchtower 3 has been released end Jan and introduced breaking change across the board that might
diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json
index 95e3d2dfd1..9c96c44a51 100644
--- a/generated/provider_dependencies.json
+++ b/generated/provider_dependencies.json
@@ -29,10 +29,6 @@
"boto3>=1.28.0",
"botocore>=1.31.0",
"jsonpath_ng>=1.5.3",
- "mypy-boto3-appflow>=1.28.0",
- "mypy-boto3-rds>=1.28.0",
- "mypy-boto3-redshift-data>=1.28.0",
- "mypy-boto3-s3>=1.28.0",
"redshift_connector>=2.0.888",
"sqlalchemy_redshift>=0.8.6",
"watchtower~=2.0.1"
diff --git a/setup.py b/setup.py
index 483ac2da84..5967552364 100644
--- a/setup.py
+++ b/setup.py
@@ -384,7 +384,6 @@ mypy_dependencies = [
# Make sure to upgrade the mypy version in update-common-sql-api-stubs in .pre-commit-config.yaml
# when you upgrade it here !!!!
"mypy==1.2.0",
- "types-boto",
"types-certifi",
"types-croniter",
"types-Deprecated",
@@ -405,50 +404,86 @@ mypy_dependencies = [
"types-PyYAML",
]
-# Dependencies needed for development only
-devel_only = [
+# make sure to update providers/amazon/provider.yaml botocore min version when you update it here
+_MIN_BOTO3_VERSION = "1.28.0"
+
+_devel_only_amazon = [
"aws_xray_sdk",
- "beautifulsoup4>=4.7.1",
- "black",
- "blinker",
- "bowler",
- "click>=8.0",
- "coverage",
+ "moto[cloudformation, glue]>=4.0",
+ f"mypy-boto3-rds>={_MIN_BOTO3_VERSION}",
+ f"mypy-boto3-redshift-data>={_MIN_BOTO3_VERSION}",
+ f"mypy-boto3-s3>={_MIN_BOTO3_VERSION}",
+ f"mypy-boto3-appflow>={_MIN_BOTO3_VERSION}",
+]
+
+_devel_only_azure = [
+ "pywinrm",
+]
+
+_devel_only_breeze = [
"filelock",
- "gitpython",
+]
+
+_devel_only_debuggers = [
"ipdb",
- "jira",
- "jsondiff",
- "jsonpath_ng>=1.5.3",
- "mongomock",
- "moto[cloudformation, glue]>=4.0",
- "paramiko",
+]
+
+_devel_only_devscripts = [
+ "click>=8.0",
+ "gitpython",
"pipdeptree",
- "pre-commit",
- "pypsrp",
"pygithub",
+ "rich-click>=1.5",
+ "semver",
+ "towncrier",
+ "twine",
+ "wheel",
+]
+
+_devel_only_mongo = [
+ "mongomock",
+]
+
+_devel_only_sentry = [
+ "blinker",
+]
+
+_devel_only_static_checks = [
+ "pre-commit",
+ "black",
+ "ruff>=0.0.219",
+ "yamllint",
+]
+
+_devel_only_tests = [
+ "aioresponses",
+ "beautifulsoup4>=4.7.1",
+ "coverage>=7.2",
"pytest",
"pytest-asyncio",
"pytest-capture-warnings",
"pytest-cov",
+ "pytest-httpx",
"pytest-instafail",
"pytest-mock",
"pytest-rerunfailures",
"pytest-timeouts",
"pytest-xdist",
- "python-jose",
- "pywinrm",
- "pytest-httpx",
"requests_mock",
- "rich-click>=1.5",
- "ruff>=0.0.219",
- "semver",
"time-machine",
- "towncrier",
- "twine",
- "wheel",
- "yamllint",
- "aioresponses",
+]
+
+# Dependencies needed for development only
+devel_only = [
+ *_devel_only_amazon,
+ *_devel_only_azure,
+ *_devel_only_breeze,
+ *_devel_only_debuggers,
+ *_devel_only_devscripts,
+ *_devel_only_mongo,
+ *_devel_only_sentry,
+ *_devel_only_static_checks,
+ *_devel_only_tests,
]
aiobotocore = [