You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by as...@apache.org on 2021/02/17 09:45:44 UTC
[airflow] branch master updated: Improved unit tests for
open_maybe_zipped function. (#14114)
This is an automated email from the ASF dual-hosted git repository.
ash pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/master by this push:
new faa6668 Improved unit tests for open_maybe_zipped function. (#14114)
faa6668 is described below
commit faa6668f3e6dba1d835840d6a134c90a707fa8c7
Author: Lev Himmelfarb <le...@boylesoftware.com>
AuthorDate: Wed Feb 17 04:45:17 2021 -0500
Improved unit tests for open_maybe_zipped function. (#14114)
Implemented "real" tests for open_maybe_zipped that test file content
reading without relying on mocks.
---
tests/utils/test_dag_processing.py | 63 -------------------------------
tests/utils/test_file.py | 77 ++++++++++++++++++++++++++++++++++++++
2 files changed, 77 insertions(+), 63 deletions(-)
diff --git a/tests/utils/test_dag_processing.py b/tests/utils/test_dag_processing.py
index c009d54..26e9890 100644
--- a/tests/utils/test_dag_processing.py
+++ b/tests/utils/test_dag_processing.py
@@ -42,7 +42,6 @@ from airflow.utils.dag_processing import (
DagParsingSignal,
DagParsingStat,
)
-from airflow.utils.file import correct_maybe_zipped, open_maybe_zipped
from airflow.utils.session import create_session
from airflow.utils.state import State
from tests.core.test_logging_config import SETTINGS_FILE_VALID, settings_context
@@ -537,65 +536,3 @@ class TestDagFileProcessorAgent(unittest.TestCase):
processor_agent._process.join()
assert os.path.isfile(log_file_loc)
-
-
-class TestCorrectMaybeZipped(unittest.TestCase):
- @mock.patch("zipfile.is_zipfile")
- def test_correct_maybe_zipped_normal_file(self, mocked_is_zipfile):
- path = '/path/to/some/file.txt'
- mocked_is_zipfile.return_value = False
-
- dag_folder = correct_maybe_zipped(path)
-
- assert dag_folder == path
-
- @mock.patch("zipfile.is_zipfile")
- def test_correct_maybe_zipped_normal_file_with_zip_in_name(self, mocked_is_zipfile):
- path = '/path/to/fakearchive.zip.other/file.txt'
- mocked_is_zipfile.return_value = False
-
- dag_folder = correct_maybe_zipped(path)
-
- assert dag_folder == path
-
- @mock.patch("zipfile.is_zipfile")
- def test_correct_maybe_zipped_archive(self, mocked_is_zipfile):
- path = '/path/to/archive.zip/deep/path/to/file.txt'
- mocked_is_zipfile.return_value = True
-
- dag_folder = correct_maybe_zipped(path)
-
- assert mocked_is_zipfile.call_count == 1
- (args, kwargs) = mocked_is_zipfile.call_args_list[0]
- assert '/path/to/archive.zip' == args[0]
-
- assert dag_folder == '/path/to/archive.zip'
-
-
-class TestOpenMaybeZipped(unittest.TestCase):
- def test_open_maybe_zipped_normal_file(self):
- with mock.patch('builtins.open', mock.mock_open(read_data="data")) as mock_file:
- open_maybe_zipped('/path/to/some/file.txt')
- mock_file.assert_called_once_with('/path/to/some/file.txt', mode='r')
-
- def test_open_maybe_zipped_normal_file_with_zip_in_name(self):
- path = '/path/to/fakearchive.zip.other/file.txt'
- with mock.patch('builtins.open', mock.mock_open(read_data="data")) as mock_file:
- open_maybe_zipped(path)
- mock_file.assert_called_once_with(path, mode='r')
-
- @mock.patch("zipfile.is_zipfile")
- @mock.patch("zipfile.ZipFile")
- @mock.patch("io.TextIOWrapper")
- def test_open_maybe_zipped_archive(self, mocked_text_io_wrapper, mocked_zip_file, mocked_is_zipfile):
- mocked_is_zipfile.return_value = True
- open_return_value = mock.mock_open(read_data="data")
- instance = mocked_zip_file.return_value
- instance.open.return_value = open_return_value
-
- open_maybe_zipped('/path/to/archive.zip/deep/path/to/file.txt')
-
- mocked_text_io_wrapper.assert_called_once_with(open_return_value)
- mocked_is_zipfile.assert_called_once_with('/path/to/archive.zip')
- mocked_zip_file.assert_called_once_with('/path/to/archive.zip', mode='r')
- instance.open.assert_called_once_with('deep/path/to/file.txt')
diff --git a/tests/utils/test_file.py b/tests/utils/test_file.py
new file mode 100644
index 0000000..c4a76cf
--- /dev/null
+++ b/tests/utils/test_file.py
@@ -0,0 +1,77 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os.path
+import unittest
+from unittest import mock
+
+from airflow.utils.file import correct_maybe_zipped, open_maybe_zipped
+from tests.models import TEST_DAGS_FOLDER
+
+
+class TestCorrectMaybeZipped(unittest.TestCase):
+ @mock.patch("zipfile.is_zipfile")
+ def test_correct_maybe_zipped_normal_file(self, mocked_is_zipfile):
+ path = '/path/to/some/file.txt'
+ mocked_is_zipfile.return_value = False
+
+ dag_folder = correct_maybe_zipped(path)
+
+ assert dag_folder == path
+
+ @mock.patch("zipfile.is_zipfile")
+ def test_correct_maybe_zipped_normal_file_with_zip_in_name(self, mocked_is_zipfile):
+ path = '/path/to/fakearchive.zip.other/file.txt'
+ mocked_is_zipfile.return_value = False
+
+ dag_folder = correct_maybe_zipped(path)
+
+ assert dag_folder == path
+
+ @mock.patch("zipfile.is_zipfile")
+ def test_correct_maybe_zipped_archive(self, mocked_is_zipfile):
+ path = '/path/to/archive.zip/deep/path/to/file.txt'
+ mocked_is_zipfile.return_value = True
+
+ dag_folder = correct_maybe_zipped(path)
+
+ assert mocked_is_zipfile.call_count == 1
+ (args, kwargs) = mocked_is_zipfile.call_args_list[0]
+ assert '/path/to/archive.zip' == args[0]
+
+ assert dag_folder == '/path/to/archive.zip'
+
+
+class TestOpenMaybeZipped(unittest.TestCase):
+ def test_open_maybe_zipped_normal_file(self):
+ test_file_path = os.path.join(TEST_DAGS_FOLDER, "no_dags.py")
+ with open_maybe_zipped(test_file_path, 'r') as test_file:
+ content = test_file.read()
+ assert isinstance(content, str)
+
+ def test_open_maybe_zipped_normal_file_with_zip_in_name(self):
+ path = '/path/to/fakearchive.zip.other/file.txt'
+ with mock.patch('builtins.open', mock.mock_open(read_data="data")) as mock_file:
+ open_maybe_zipped(path)
+ mock_file.assert_called_once_with(path, mode='r')
+
+ def test_open_maybe_zipped_archive(self):
+ test_file_path = os.path.join(TEST_DAGS_FOLDER, "test_zip.zip", "test_zip.py")
+ with open_maybe_zipped(test_file_path, 'r') as test_file:
+ content = test_file.read()
+ assert isinstance(content, str)