You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ep...@apache.org on 2023/08/08 17:43:40 UTC
[airflow] 16/37: Allows to choose SSL context for IMAP provider (#33108)
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch v2-7-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 38fc9cd823feafd8ec61d5d5c7eddb9e9162f755
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Fri Aug 4 14:28:35 2023 +0200
Allows to choose SSL context for IMAP provider (#33108)
* Allows to choose SSL context for IMAP provider
This change add two options to choose from when SSL IMAP connection is created:
* default - for balance between compatibility and security
* none - in case compatibility with existing infrastructure is preferred
The fallback is:
* The Airflow "email", "ssl_context"
* "default"
Co-authored-by: Ephraim Anierobi <sp...@gmail.com>
(cherry picked from commit 52ca7bfc988f4c9b608f544bc3e9524fd6564639)
---
airflow/providers/imap/CHANGELOG.rst | 12 ++++
airflow/providers/imap/hooks/imap.py | 37 +++++++++---
airflow/providers/imap/provider.yaml | 23 ++++++++
.../configurations-ref.rst | 18 ++++++
docs/apache-airflow-providers-imap/index.rst | 1 +
docs/apache-airflow/configurations-ref.rst | 1 +
tests/providers/imap/hooks/test_imap.py | 69 +++++++++++++++++++++-
7 files changed, 150 insertions(+), 11 deletions(-)
diff --git a/airflow/providers/imap/CHANGELOG.rst b/airflow/providers/imap/CHANGELOG.rst
index 24cf83e5b8..dc5ae11b22 100644
--- a/airflow/providers/imap/CHANGELOG.rst
+++ b/airflow/providers/imap/CHANGELOG.rst
@@ -26,6 +26,18 @@
Changelog
---------
+In case of IMAP SSL connection, the context now uses the "default" context
+
+The "default" context is Python's ``default_ssl_context`` instead of previously used "none". The
+``default_ssl_context`` provides a balance between security and compatibility but in some cases,
+when certificates are old, self-signed or misconfigured, it might not work. This can be configured
+by setting "ssl_context" in "imap" configuration of the provider. If it is not explicitly set,
+it will default to "email", "ssl_context" setting in Airflow.
+
+Setting it to "none" brings back the "none" setting that was used in previous versions of the provider,
+but it is not recommended due to security reasons and this setting disables validation
+of certificates and allows MITM attacks.
+
3.2.2
.....
diff --git a/airflow/providers/imap/hooks/imap.py b/airflow/providers/imap/hooks/imap.py
index 4a00e6965a..6523c61cf7 100644
--- a/airflow/providers/imap/hooks/imap.py
+++ b/airflow/providers/imap/hooks/imap.py
@@ -26,6 +26,7 @@ import email
import imaplib
import os
import re
+import ssl
from typing import Any, Iterable
from airflow.exceptions import AirflowException
@@ -78,16 +79,34 @@ class ImapHook(BaseHook):
return self
def _build_client(self, conn: Connection) -> imaplib.IMAP4_SSL | imaplib.IMAP4:
- IMAP: type[imaplib.IMAP4_SSL] | type[imaplib.IMAP4]
- if conn.extra_dejson.get("use_ssl", True):
- IMAP = imaplib.IMAP4_SSL
- else:
- IMAP = imaplib.IMAP4
-
- if conn.port:
- mail_client = IMAP(conn.host, conn.port)
+ mail_client: imaplib.IMAP4_SSL | imaplib.IMAP4
+ use_ssl = conn.extra_dejson.get("use_ssl", True)
+ if use_ssl:
+ from airflow.configuration import conf
+
+ ssl_context_string = conf.get("imap", "SSL_CONTEXT", fallback=None)
+ if ssl_context_string is None:
+ ssl_context_string = conf.get("email", "SSL_CONTEXT", fallback=None)
+ if ssl_context_string is None:
+ ssl_context_string = "default"
+ if ssl_context_string == "default":
+ ssl_context = ssl.create_default_context()
+ elif ssl_context_string == "none":
+ ssl_context = None
+ else:
+ raise RuntimeError(
+ f"The email.ssl_context configuration variable must "
+ f"be set to 'default' or 'none' and is '{ssl_context_string}'."
+ )
+ if conn.port:
+ mail_client = imaplib.IMAP4_SSL(conn.host, conn.port, ssl_context=ssl_context)
+ else:
+ mail_client = imaplib.IMAP4_SSL(conn.host, ssl_context=ssl_context)
else:
- mail_client = IMAP(conn.host)
+ if conn.port:
+ mail_client = imaplib.IMAP4(conn.host, conn.port)
+ else:
+ mail_client = imaplib.IMAP4(conn.host)
return mail_client
diff --git a/airflow/providers/imap/provider.yaml b/airflow/providers/imap/provider.yaml
index 997880d4bd..e7d427a813 100644
--- a/airflow/providers/imap/provider.yaml
+++ b/airflow/providers/imap/provider.yaml
@@ -62,3 +62,26 @@ hooks:
connection-types:
- hook-class-name: airflow.providers.imap.hooks.imap.ImapHook
connection-type: imap
+
+config:
+ imap:
+ description: "Options for IMAP provider."
+ options:
+ ssl_context:
+ description: |
+ ssl context to use when using SMTP and IMAP SSL connections. By default, the context is "default"
+ which sets it to ``ssl.create_default_context()`` which provides the right balance between
+ compatibility and security, it however requires that certificates in your operating system are
+ updated and that SMTP/IMAP servers of yours have valid certificates that have corresponding public
+ keys installed on your machines. You can switch it to "none" if you want to disable checking
+ of the certificates, but it is not recommended as it allows MITM (man-in-the-middle) attacks
+ if your infrastructure is not sufficiently secured. It should only be set temporarily while you
+ are fixing your certificate configuration. This can be typically done by upgrading to newer
+ version of the operating system you run Airflow components on,by upgrading/refreshing proper
+ certificates in the OS or by updating certificates for your mail servers.
+ If you do not set this option explicitly, it will use Airflow "email.ssl_context" configuration,
+ but if this configuration is not present, it will use "default" value.
+ type: string
+ version_added: 3.3.0
+ example: "default"
+ default: ~
diff --git a/docs/apache-airflow-providers-imap/configurations-ref.rst b/docs/apache-airflow-providers-imap/configurations-ref.rst
new file mode 100644
index 0000000000..5885c9d91b
--- /dev/null
+++ b/docs/apache-airflow-providers-imap/configurations-ref.rst
@@ -0,0 +1,18 @@
+ .. Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ .. http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+.. include:: ../exts/includes/providers-configurations-ref.rst
diff --git a/docs/apache-airflow-providers-imap/index.rst b/docs/apache-airflow-providers-imap/index.rst
index ef8e28b9e0..60f86f1ece 100644
--- a/docs/apache-airflow-providers-imap/index.rst
+++ b/docs/apache-airflow-providers-imap/index.rst
@@ -34,6 +34,7 @@
:maxdepth: 1
:caption: References
+ Configuration <configurations-ref>
Connection types <connections/imap>
Python API <_api/airflow/providers/imap/index>
diff --git a/docs/apache-airflow/configurations-ref.rst b/docs/apache-airflow/configurations-ref.rst
index 42ff5b9a6b..43173fe6ac 100644
--- a/docs/apache-airflow/configurations-ref.rst
+++ b/docs/apache-airflow/configurations-ref.rst
@@ -39,6 +39,7 @@ in the provider's documentation. The pre-installed providers that you may want t
* :doc:`Configuration Reference for Apache Hive Provider <apache-airflow-providers-apache-hive:configurations-ref>`
* :doc:`Configuration Reference for CNCF Kubernetes Provider <apache-airflow-providers-cncf-kubernetes:configurations-ref>`
* :doc:`Configuration Reference for SMTP Provider <apache-airflow-providers-smtp:configurations-ref>`
+* :doc:`Configuration Reference for IMAP Provider <apache-airflow-providers-imap:configurations-ref>`
.. note::
For more information see :doc:`/howto/set-config`.
diff --git a/tests/providers/imap/hooks/test_imap.py b/tests/providers/imap/hooks/test_imap.py
index c2be420be6..5e57d7196e 100644
--- a/tests/providers/imap/hooks/test_imap.py
+++ b/tests/providers/imap/hooks/test_imap.py
@@ -27,6 +27,7 @@ from airflow.exceptions import AirflowException
from airflow.models import Connection
from airflow.providers.imap.hooks.imap import ImapHook
from airflow.utils import db
+from tests.test_utils.config import conf_vars
imaplib_string = "airflow.providers.imap.hooks.imap.imaplib"
open_string = "airflow.providers.imap.hooks.imap.open"
@@ -85,13 +86,77 @@ class TestImapHook:
)
@patch(imaplib_string)
- def test_connect_and_disconnect(self, mock_imaplib):
+ @patch("ssl.create_default_context")
+ def test_connect_and_disconnect(self, create_default_context, mock_imaplib):
mock_conn = _create_fake_imap(mock_imaplib)
with ImapHook():
pass
- mock_imaplib.IMAP4_SSL.assert_called_once_with("imap_server_address", 1993)
+ assert create_default_context.called
+ mock_imaplib.IMAP4_SSL.assert_called_once_with(
+ "imap_server_address", 1993, ssl_context=create_default_context.return_value
+ )
+ mock_conn.login.assert_called_once_with("imap_user", "imap_password")
+ assert mock_conn.logout.call_count == 1
+
+ @patch(imaplib_string)
+ @patch("ssl.create_default_context")
+ def test_connect_and_disconnect_imap_ssl_context_none(self, create_default_context, mock_imaplib):
+ mock_conn = _create_fake_imap(mock_imaplib)
+
+ with conf_vars({("imap", "ssl_context"): "none"}):
+ with ImapHook():
+ pass
+
+ assert not create_default_context.called
+ mock_imaplib.IMAP4_SSL.assert_called_once_with("imap_server_address", 1993, ssl_context=None)
+ mock_conn.login.assert_called_once_with("imap_user", "imap_password")
+ assert mock_conn.logout.call_count == 1
+
+ @patch(imaplib_string)
+ @patch("ssl.create_default_context")
+ def test_connect_and_disconnect_imap_ssl_context_default(self, create_default_context, mock_imaplib):
+ mock_conn = _create_fake_imap(mock_imaplib)
+
+ with conf_vars({("imap", "ssl_context"): "default"}):
+ with ImapHook():
+ pass
+
+ assert create_default_context.called
+ mock_imaplib.IMAP4_SSL.assert_called_once_with(
+ "imap_server_address", 1993, ssl_context=create_default_context.return_value
+ )
+ mock_conn.login.assert_called_once_with("imap_user", "imap_password")
+ assert mock_conn.logout.call_count == 1
+
+ @patch(imaplib_string)
+ @patch("ssl.create_default_context")
+ def test_connect_and_disconnect_email_ssl_context_none(self, create_default_context, mock_imaplib):
+ mock_conn = _create_fake_imap(mock_imaplib)
+
+ with conf_vars({("email", "ssl_context"): "none"}):
+ with ImapHook():
+ pass
+
+ assert not create_default_context.called
+ mock_imaplib.IMAP4_SSL.assert_called_once_with("imap_server_address", 1993, ssl_context=None)
+ mock_conn.login.assert_called_once_with("imap_user", "imap_password")
+ assert mock_conn.logout.call_count == 1
+
+ @patch(imaplib_string)
+ @patch("ssl.create_default_context")
+ def test_connect_and_disconnect_imap_ssl_context_override(self, create_default_context, mock_imaplib):
+ mock_conn = _create_fake_imap(mock_imaplib)
+
+ with conf_vars({("email", "ssl_context"): "none", ("imap", "ssl_context"): "default"}):
+ with ImapHook():
+ pass
+
+ assert create_default_context.called
+ mock_imaplib.IMAP4_SSL.assert_called_once_with(
+ "imap_server_address", 1993, ssl_context=create_default_context.return_value
+ )
mock_conn.login.assert_called_once_with("imap_user", "imap_password")
assert mock_conn.logout.call_count == 1