You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2022/09/21 19:52:47 UTC
[airflow] branch main updated: Remove Amazon S3 Connection Type (#25980)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new cf73cb79d7 Remove Amazon S3 Connection Type (#25980)
cf73cb79d7 is described below
commit cf73cb79d7aba49d3934864acdd9ce1599836d0f
Author: Andrey Anshin <An...@taragol.is>
AuthorDate: Wed Sep 21 23:52:35 2022 +0400
Remove Amazon S3 Connection Type (#25980)
---
airflow/providers/amazon/CHANGELOG.rst | 8 ++++++++
airflow/providers/amazon/aws/hooks/base_aws.py | 2 +-
airflow/providers/amazon/aws/hooks/s3.py | 3 ---
airflow/providers/amazon/aws/utils/connection_wrapper.py | 14 ++++++++++++--
airflow/providers/amazon/provider.yaml | 2 --
docs/apache-airflow-providers-amazon/connections/aws.rst | 2 +-
.../logging/s3-task-handler.rst | 4 ++--
.../providers/amazon/aws/utils/test_connection_wrapper.py | 12 +++++++++++-
8 files changed, 35 insertions(+), 12 deletions(-)
diff --git a/airflow/providers/amazon/CHANGELOG.rst b/airflow/providers/amazon/CHANGELOG.rst
index 059b8123c6..82b19192de 100644
--- a/airflow/providers/amazon/CHANGELOG.rst
+++ b/airflow/providers/amazon/CHANGELOG.rst
@@ -24,6 +24,14 @@
Changelog
---------
+.. warning::
+ In this version of provider Amazon S3 Connection (``conn_type="s3"``) removed due to the fact that it was always
+ an alias to :ref:`Amazon Web Services Connection <howto/connection:aws>` (``conn_type="aws"``).
+ In practice the only impact is you won't be able to ``test`` the connection in the web UI / API.
+ In order to restore ability to test connection you need to change connection type from **Amazon S3** (``conn_type="s3"``)
+ to **Amazon Web Services** (``conn_type="aws"``) manually.
+
+
5.1.0
.....
diff --git a/airflow/providers/amazon/aws/hooks/base_aws.py b/airflow/providers/amazon/aws/hooks/base_aws.py
index f6ce992e41..1bc2e99ee7 100644
--- a/airflow/providers/amazon/aws/hooks/base_aws.py
+++ b/airflow/providers/amazon/aws/hooks/base_aws.py
@@ -20,7 +20,7 @@ This module contains Base AWS Hook.
.. seealso::
For more information on how to use this hook, take a look at the guide:
- :ref:`howto/connection:AWSHook`
+ :ref:`howto/connection:aws`
"""
from __future__ import annotations
diff --git a/airflow/providers/amazon/aws/hooks/s3.py b/airflow/providers/amazon/aws/hooks/s3.py
index 42a360d48f..ef183d59ae 100644
--- a/airflow/providers/amazon/aws/hooks/s3.py
+++ b/airflow/providers/amazon/aws/hooks/s3.py
@@ -114,9 +114,6 @@ class S3Hook(AwsBaseHook):
:class:`~airflow.providers.amazon.aws.hooks.base_aws.AwsBaseHook`
"""
- conn_type = 's3'
- hook_name = 'Amazon S3'
-
def __init__(
self,
aws_conn_id: str | None = AwsBaseHook.default_conn_name,
diff --git a/airflow/providers/amazon/aws/utils/connection_wrapper.py b/airflow/providers/amazon/aws/utils/connection_wrapper.py
index 18b057b18b..d5a38cacf4 100644
--- a/airflow/providers/amazon/aws/utils/connection_wrapper.py
+++ b/airflow/providers/amazon/aws/utils/connection_wrapper.py
@@ -128,9 +128,19 @@ class AwsConnectionWrapper(LoggingMixin):
self.password = conn.password
self.extra_config = deepcopy(conn.extra_dejson)
- if self.conn_type != "aws":
+ if self.conn_type.lower() == "s3":
warnings.warn(
- f"{self.conn_repr} expected connection type 'aws', got {self.conn_type!r}.",
+ f"{self.conn_repr} has connection type 's3', "
+ "which has been replaced by connection type 'aws'. "
+ "Please update your connection to have `conn_type='aws'`.",
+ DeprecationWarning,
+ stacklevel=2,
+ )
+ elif self.conn_type != "aws":
+ warnings.warn(
+ f"{self.conn_repr} expected connection type 'aws', got {self.conn_type!r}. "
+ "This connection might not work correctly. "
+ "Please use Amazon Web Services Connection type.",
UserWarning,
stacklevel=2,
)
diff --git a/airflow/providers/amazon/provider.yaml b/airflow/providers/amazon/provider.yaml
index 50b15760e9..2583a3f19c 100644
--- a/airflow/providers/amazon/provider.yaml
+++ b/airflow/providers/amazon/provider.yaml
@@ -548,8 +548,6 @@ extra-links:
- airflow.providers.amazon.aws.links.logs.CloudWatchEventsLink
connection-types:
- - hook-class-name: airflow.providers.amazon.aws.hooks.s3.S3Hook
- connection-type: s3
- hook-class-name: airflow.providers.amazon.aws.hooks.base_aws.AwsGenericHook
connection-type: aws
- hook-class-name: airflow.providers.amazon.aws.hooks.emr.EmrHook
diff --git a/docs/apache-airflow-providers-amazon/connections/aws.rst b/docs/apache-airflow-providers-amazon/connections/aws.rst
index 9223db8652..a57ab8e30a 100644
--- a/docs/apache-airflow-providers-amazon/connections/aws.rst
+++ b/docs/apache-airflow-providers-amazon/connections/aws.rst
@@ -15,7 +15,7 @@
specific language governing permissions and limitations
under the License.
-.. _howto/connection:AWSHook:
+.. _howto/connection:aws:
Amazon Web Services Connection
==============================
diff --git a/docs/apache-airflow-providers-amazon/logging/s3-task-handler.rst b/docs/apache-airflow-providers-amazon/logging/s3-task-handler.rst
index bc120887fe..50e7d7e9f4 100644
--- a/docs/apache-airflow-providers-amazon/logging/s3-task-handler.rst
+++ b/docs/apache-airflow-providers-amazon/logging/s3-task-handler.rst
@@ -37,11 +37,11 @@ To enable this feature, ``airflow.cfg`` must be configured as follows:
# id that provides access to the storage location.
remote_logging = True
remote_base_log_folder = s3://my-bucket/path/to/logs
- remote_log_conn_id = MyS3Conn
+ remote_log_conn_id = my_s3_conn
# Use server-side encryption for logs stored in S3
encrypt_s3_logs = False
-In the above example, Airflow will try to use ``S3Hook('MyS3Conn')``.
+In the above example, Airflow will try to use ``S3Hook(aws_conn_id='my_s3_conn')``.
You can also use `LocalStack <https://localstack.cloud/>`_ to emulate Amazon S3 locally.
To configure it, you must additionally set the endpoint url to point to your local stack.
diff --git a/tests/providers/amazon/aws/utils/test_connection_wrapper.py b/tests/providers/amazon/aws/utils/test_connection_wrapper.py
index e024132bdc..3540f86e61 100644
--- a/tests/providers/amazon/aws/utils/test_connection_wrapper.py
+++ b/tests/providers/amazon/aws/utils/test_connection_wrapper.py
@@ -77,13 +77,23 @@ class TestAwsConnectionWrapper:
wrap_conn = AwsConnectionWrapper(conn=mock_connection_factory(conn_type=conn_type))
assert wrap_conn.conn_type == "aws"
- @pytest.mark.parametrize("conn_type", ["AWS", "boto3", "s3", "emr", "google", "google-cloud-platform"])
+ @pytest.mark.parametrize("conn_type", ["AWS", "boto3", "emr", "google", "google-cloud-platform"])
def test_unexpected_aws_connection_type(self, conn_type):
warning_message = f"expected connection type 'aws', got '{conn_type}'"
with pytest.warns(UserWarning, match=warning_message):
wrap_conn = AwsConnectionWrapper(conn=mock_connection_factory(conn_type=conn_type))
assert wrap_conn.conn_type == conn_type
+ @pytest.mark.parametrize("conn_type", ["s3", "S3"])
+ def test_deprecated_s3_connection_type(self, conn_type):
+ warning_message = (
+ r".* has connection type 's3', which has been replaced by connection type 'aws'\. "
+ r"Please update your connection to have `conn_type='aws'`."
+ )
+ with pytest.warns(DeprecationWarning, match=warning_message):
+ wrap_conn = AwsConnectionWrapper(conn=mock_connection_factory(conn_type=conn_type))
+ assert wrap_conn.conn_type == conn_type
+
@pytest.mark.parametrize("aws_session_token", [None, "mock-aws-session-token"])
@pytest.mark.parametrize("aws_secret_access_key", ["mock-aws-secret-access-key"])
@pytest.mark.parametrize("aws_access_key_id", ["mock-aws-access-key-id"])