You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2025/04/06 11:38:52 UTC
(airflow) 01/01: Change provider-specific dependencies to refer to providers
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch optional-extras-moved-to-providers
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit ce1b53b2955bcf0ec3ecee042a2038d79899e917
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Sun Apr 6 13:29:16 2025 +0200
Change provider-specific dependencies to refer to providers
With the recent `uv` move, we can move some of the optional dependencies
to providers - i.e. rather than refer to specific dependenciesm we can
refer to the optional dependencies of the providers. This was not easy
before - because with `pip` for development dependencies it would have
install the provider itself instead of the dependencies, but with uv
workspaces this is perfectly fine as uv will find the provider in
the workspace and will install the provider sources and the optional
dependencies when given extra is used.
This avoids duplication in the core's dependencies and allows to
depend airflow core noot on specific dependencies but on the
optional dependencies of provider installed, which is as it should be.
This change also bumps the dependencies of amazon provider that
are affected by this change in an attempt to make "resolution too
deep" problems.
---
airflow-core/pyproject.toml | 7 +++----
providers/amazon/README.rst | 14 +++++++-------
providers/amazon/pyproject.toml | 10 +++++-----
.../src/airflow/providers/amazon/get_provider_info.py | 10 +++++-----
4 files changed, 20 insertions(+), 21 deletions(-)
diff --git a/airflow-core/pyproject.toml b/airflow-core/pyproject.toml
index dbfe3b4978a..1373ac5c47c 100644
--- a/airflow-core/pyproject.toml
+++ b/airflow-core/pyproject.toml
@@ -154,7 +154,7 @@ dependencies = [
# TODO: We can remove it once boto3 and aiobotocore both have compatible botocore version or
# boto3 have native aync support and we move away from aio aiobotocore
"aiobotocore" = [
- "aiobotocore>=2.13.0",
+ "apache-airflow-providers-amazon[aiobotocore]",
]
"async" = [
"eventlet>=0.37.0",
@@ -165,13 +165,12 @@ dependencies = [
"atlasclient>=0.1.2",
]
"apache-webhdfs" = [
- "hdfs[avro,dataframe,kerberos]>=2.0.4",
+ "apache-airflow-providers-hdfs",
]
"cgroups" = [
"cgroupspy>=0.2.2",
]
"cloudpickle" = [
- # Latest version of apache-beam requires cloudpickle~=2.2.1
"cloudpickle>=2.2.1",
]
"github-enterprise" = [
@@ -212,7 +211,7 @@ dependencies = [
"s3fs" = [
# This is required for support of S3 file system which uses aiobotocore
# which can have a conflict with boto3 as mentioned in aiobotocore extra
- "s3fs>=2023.10.0",
+ "apache-airflow-providers-amazon[s3fs]",
]
"sentry" = [
"blinker>=1.1",
diff --git a/providers/amazon/README.rst b/providers/amazon/README.rst
index fb7a21632bb..e7ea3f36d10 100644
--- a/providers/amazon/README.rst
+++ b/providers/amazon/README.rst
@@ -50,26 +50,26 @@ The package supports the following python versions: 3.9,3.10,3.11,3.12
Requirements
------------
-========================================== ======================
+========================================== =====================
PIP package Version required
-========================================== ======================
+========================================== =====================
``apache-airflow`` ``>=2.9.0``
``apache-airflow-providers-common-compat`` ``>=1.6.0``
``apache-airflow-providers-common-sql`` ``>=1.20.0``
``apache-airflow-providers-http``
``boto3`` ``>=1.37.0``
-``botocore`` ``>=1.34.90``
+``botocore`` ``>=1.37.0``
``inflection`` ``>=0.5.1``
-``watchtower`` ``>=3.0.0,!=3.3.0,<4``
+``watchtower`` ``>=3.3.1,<4``
``jsonpath_ng`` ``>=1.5.3``
-``redshift_connector`` ``>=2.0.918``
+``redshift_connector`` ``>=2.1.3``
``asgiref`` ``>=2.3.0``
-``PyAthena`` ``>=3.0.10``
+``PyAthena`` ``>=3.10.0``
``jmespath`` ``>=0.7.0``
``python3-saml`` ``>=1.16.0``
``xmlsec`` ``>=1.3.14,!=1.3.15``
``sagemaker-studio`` ``>=1.0.9``
-========================================== ======================
+========================================== =====================
Cross provider package dependencies
-----------------------------------
diff --git a/providers/amazon/pyproject.toml b/providers/amazon/pyproject.toml
index 143c2829865..18b0853391a 100644
--- a/providers/amazon/pyproject.toml
+++ b/providers/amazon/pyproject.toml
@@ -65,14 +65,14 @@ dependencies = [
# of candidates to consider. Make sure to configure boto3 version here as well as in all the tools below
# in the `devel-dependencies` section to be the same minimum version.
"boto3>=1.37.0",
- "botocore>=1.34.90",
+ "botocore>=1.37.0",
"inflection>=0.5.1",
# Allow a wider range of watchtower versions for flexibility among users
- "watchtower>=3.0.0,!=3.3.0,<4",
+ "watchtower>=3.3.1,<4",
"jsonpath_ng>=1.5.3",
- "redshift_connector>=2.0.918",
+ "redshift_connector>=2.1.3",
"asgiref>=2.3.0",
- "PyAthena>=3.0.10",
+ "PyAthena>=3.10.0",
"jmespath>=0.7.0",
"python3-saml>=1.16.0",
# python3-saml is dependent on xmlsec and seems they do not pin it, pinning here would be needed
@@ -95,7 +95,7 @@ dependencies = [
# TODO: We can remove it once boto3 and aiobotocore both have compatible botocore version or
# boto3 have native async support and we move away from aio aiobotocore
"aiobotocore" = [
- "aiobotocore[boto3]>=2.13.0",
+ "aiobotocore[boto3]>=2.20.0",
]
"cncf.kubernetes" = [
"apache-airflow-providers-cncf-kubernetes>=7.2.0",
diff --git a/providers/amazon/src/airflow/providers/amazon/get_provider_info.py b/providers/amazon/src/airflow/providers/amazon/get_provider_info.py
index e4c96685b6f..37cd4acbb6a 100644
--- a/providers/amazon/src/airflow/providers/amazon/get_provider_info.py
+++ b/providers/amazon/src/airflow/providers/amazon/get_provider_info.py
@@ -1376,13 +1376,13 @@ def get_provider_info():
"apache-airflow-providers-common-sql>=1.20.0",
"apache-airflow-providers-http",
"boto3>=1.37.0",
- "botocore>=1.34.90",
+ "botocore>=1.37.0",
"inflection>=0.5.1",
- "watchtower>=3.0.0,!=3.3.0,<4",
+ "watchtower>=3.3.1,<4",
"jsonpath_ng>=1.5.3",
- "redshift_connector>=2.0.918",
+ "redshift_connector>=2.1.3",
"asgiref>=2.3.0",
- "PyAthena>=3.0.10",
+ "PyAthena>=3.10.0",
"jmespath>=0.7.0",
"python3-saml>=1.16.0",
"xmlsec!=1.3.15,>=1.3.14",
@@ -1390,7 +1390,7 @@ def get_provider_info():
],
"optional-dependencies": {
"pandas": ["pandas>=2.1.2,<2.2"],
- "aiobotocore": ["aiobotocore[boto3]>=2.13.0"],
+ "aiobotocore": ["aiobotocore[boto3]>=2.20.0"],
"cncf.kubernetes": ["apache-airflow-providers-cncf-kubernetes>=7.2.0"],
"s3fs": ["s3fs>=2023.10.0"],
"python3-saml": ["python3-saml>=1.16.0"],