You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by bi...@apache.org on 2021/04/12 15:34:36 UTC
[airflow] branch master updated: Remove python2 related handlings
and dependencies (#15301)
This is an automated email from the ASF dual-hosted git repository.
binh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/master by this push:
new 6f8ab9e Remove python2 related handlings and dependencies (#15301)
6f8ab9e is described below
commit 6f8ab9e6da9b7ccd898e91f0bdf5311c7f1b8336
Author: Xinbin Huang <bi...@gmail.com>
AuthorDate: Mon Apr 12 08:34:26 2021 -0700
Remove python2 related handlings and dependencies (#15301)
* Remove python2 related handlings and dependencies
* fixup! Remove python2 related handlings and dependencies
* Keep PythonVirtualenvOperator tests for python2
* Keep __init__.py with comments
---
airflow/providers/apache/hdfs/hooks/hdfs.py | 3 +--
airflow/providers/imap/hooks/imap.py | 2 +-
tests/core/test_core.py | 13 ++++---------
tests/core/test_logging_config.py | 9 +++------
tests/operators/test_python.py | 12 ++++++------
tests/www/test_views.py | 14 ++------------
6 files changed, 17 insertions(+), 36 deletions(-)
diff --git a/airflow/providers/apache/hdfs/hooks/hdfs.py b/airflow/providers/apache/hdfs/hooks/hdfs.py
index de3f2b5..5cd3523 100644
--- a/airflow/providers/apache/hdfs/hooks/hdfs.py
+++ b/airflow/providers/apache/hdfs/hooks/hdfs.py
@@ -59,8 +59,7 @@ class HDFSHook(BaseHook):
raise ImportError(
'This HDFSHook implementation requires snakebite, but '
'snakebite is not compatible with Python 3 '
- '(as of August 2015). Please use Python 2 if you require '
- 'this hook -- or help by submitting a PR!'
+ '(as of August 2015). Please help by submitting a PR!'
)
self.hdfs_conn_id = hdfs_conn_id
self.proxy_user = proxy_user
diff --git a/airflow/providers/imap/hooks/imap.py b/airflow/providers/imap/hooks/imap.py
index 316709a..7ca4511 100644
--- a/airflow/providers/imap/hooks/imap.py
+++ b/airflow/providers/imap/hooks/imap.py
@@ -18,7 +18,7 @@
"""
This module provides everything to be able to search in mails for a specific attachment
and also to download it.
-It uses the imaplib library that is already integrated in python 2 and 3.
+It uses the imaplib library that is already integrated in python 3.
"""
import email
import imaplib
diff --git a/tests/core/test_core.py b/tests/core/test_core.py
index 061a1a0..7149112 100644
--- a/tests/core/test_core.py
+++ b/tests/core/test_core.py
@@ -22,6 +22,7 @@ import signal
import unittest
from datetime import timedelta
from time import sleep
+from unittest.mock import MagicMock
import pytest
@@ -187,23 +188,17 @@ class TestCore(unittest.TestCase):
self.fail("BashOperator's subprocess still running after stopping on timeout!")
def test_on_failure_callback(self):
- # Annoying workaround for nonlocal not existing in python 2
- data = {'called': False}
-
- def check_failure(context, test_case=self): # pylint: disable=unused-argument
- data['called'] = True
- error = context.get("exception")
- test_case.assertIsInstance(error, AirflowException)
+ mock_failure_callback = MagicMock()
op = BashOperator(
task_id='check_on_failure_callback',
bash_command="exit 1",
dag=self.dag,
- on_failure_callback=check_failure,
+ on_failure_callback=mock_failure_callback,
)
with pytest.raises(AirflowException):
op.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE, ignore_ti_state=True)
- assert data['called']
+ mock_failure_callback.assert_called_once()
def test_dryrun(self):
op = BashOperator(task_id='test_dryrun', bash_command="echo success", dag=self.dag)
diff --git a/tests/core/test_logging_config.py b/tests/core/test_logging_config.py
index b9714b7..c514b08 100644
--- a/tests/core/test_logging_config.py
+++ b/tests/core/test_logging_config.py
@@ -137,21 +137,18 @@ def settings_context(content, directory=None, name='LOGGING_CONFIG'):
settings_root = tempfile.mkdtemp()
filename = f"{SETTINGS_DEFAULT_NAME}.py"
if directory:
- # Replace slashes by dots
- module = directory.replace('/', '.') + '.' + SETTINGS_DEFAULT_NAME + '.' + name
-
- # Create the directory structure
+ # Create the directory structure with __init__.py
dir_path = os.path.join(settings_root, directory)
pathlib.Path(dir_path).mkdir(parents=True, exist_ok=True)
- # Add the __init__ for the directories
- # This is required for Python 2.7
basedir = settings_root
for part in directory.split('/'):
open(os.path.join(basedir, '__init__.py'), 'w').close()
basedir = os.path.join(basedir, part)
open(os.path.join(basedir, '__init__.py'), 'w').close()
+ # Replace slashes by dots
+ module = directory.replace('/', '.') + '.' + SETTINGS_DEFAULT_NAME + '.' + name
settings_file = os.path.join(dir_path, filename)
else:
module = SETTINGS_DEFAULT_NAME + '.' + name
diff --git a/tests/operators/test_python.py b/tests/operators/test_python.py
index 5bd78c6..3853d3a 100644
--- a/tests/operators/test_python.py
+++ b/tests/operators/test_python.py
@@ -863,18 +863,18 @@ class TestPythonVirtualenvOperator(unittest.TestCase):
else:
return 2
- def test_wrong_python_op_args(self):
- if sys.version_info[0] == 2:
- version = 3
- else:
- version = 2
-
+ def test_wrong_python_version_with_op_args(self):
def f():
pass
+ version = self._invert_python_major_version()
+
with pytest.raises(AirflowException):
self._run_as_operator(f, python_version=version, op_args=[1])
+ with pytest.raises(AirflowException):
+ self._run_as_operator(f, python_version=version, op_kwargs={"arg": 1})
+
def test_without_dill(self):
def f(a):
return a
diff --git a/tests/www/test_views.py b/tests/www/test_views.py
index fb47d65..5844512 100644
--- a/tests/www/test_views.py
+++ b/tests/www/test_views.py
@@ -318,12 +318,7 @@ class TestVariableModelView(TestBase):
set_mock.side_effect = UnicodeEncodeError
assert self.session.query(models.Variable).count() == 0
- try:
- # python 3+
- bytes_content = io.BytesIO(bytes(content, encoding='utf-8'))
- except TypeError:
- # python 2.7
- bytes_content = io.BytesIO(bytes(content))
+ bytes_content = io.BytesIO(bytes(content, encoding='utf-8'))
resp = self.client.post(
'/variable/varimport', data={'file': (bytes_content, 'test.json')}, follow_redirects=True
@@ -336,12 +331,7 @@ class TestVariableModelView(TestBase):
content = (
'{"str_key": "str_value", "int_key": 60, "list_key": [1, 2], "dict_key": {"k_a": 2, "k_b": 3}}'
)
- try:
- # python 3+
- bytes_content = io.BytesIO(bytes(content, encoding='utf-8'))
- except TypeError:
- # python 2.7
- bytes_content = io.BytesIO(bytes(content))
+ bytes_content = io.BytesIO(bytes(content, encoding='utf-8'))
resp = self.client.post(
'/variable/varimport', data={'file': (bytes_content, 'test.json')}, follow_redirects=True