You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2022/03/07 01:15:17 UTC
[airflow] branch main updated: Switch oss hook tests in alibaba-provider to use Mocks (17617) (#21992)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 73c6bf0 Switch oss hook tests in alibaba-provider to use Mocks (17617) (#21992)
73c6bf0 is described below
commit 73c6bf08780780ca5a318e74902cb05ba006e3ba
Author: Eric Gao <er...@gmail.com>
AuthorDate: Mon Mar 7 09:14:30 2022 +0800
Switch oss hook tests in alibaba-provider to use Mocks (17617) (#21992)
---
tests/providers/alibaba/cloud/hooks/test_oss.py | 123 +++++++++++++++++-------
tests/providers/alibaba/cloud/utils/oss_mock.py | 36 +++++++
2 files changed, 125 insertions(+), 34 deletions(-)
diff --git a/tests/providers/alibaba/cloud/hooks/test_oss.py b/tests/providers/alibaba/cloud/hooks/test_oss.py
index 4bebb1d..fe60893 100644
--- a/tests/providers/alibaba/cloud/hooks/test_oss.py
+++ b/tests/providers/alibaba/cloud/hooks/test_oss.py
@@ -16,53 +16,108 @@
# specific language governing permissions and limitations
# under the License.
#
-import os
import unittest
+from unittest import mock
-import oss2
-
-from airflow.exceptions import AirflowException
from airflow.providers.alibaba.cloud.hooks.oss import OSSHook
-from tests.providers.alibaba.cloud.utils.test_utils import skip_test_if_no_valid_conn_id
+from tests.providers.alibaba.cloud.utils.oss_mock import mock_oss_hook_default_project_id
-TEST_CONN_ID = os.environ.get('TEST_OSS_CONN_ID', 'oss_default')
-TEST_REGION = os.environ.get('TEST_OSS_REGION', 'us-east-1')
-TEST_BUCKET = os.environ.get('TEST_OSS_BUCKET', 'test-bucket')
+OSS_STRING = 'airflow.providers.alibaba.cloud.hooks.oss.{}'
+MOCK_OSS_CONN_ID = 'mock_id'
+MOCK_BUCKET_NAME = 'mock_bucket_name'
+MOCK_KEY = 'mock_key'
+MOCK_KEYS = ['mock_key1', 'mock_key2', 'mock_key3']
+MOCK_CONTENT = 'mock_content'
+MOCK_FILE_PATH = 'mock_file_path'
class TestOSSHook(unittest.TestCase):
def setUp(self):
- try:
- self.hook = OSSHook(region=TEST_REGION, oss_conn_id=TEST_CONN_ID)
- self.hook.object_exists(key='test-obj', bucket_name=TEST_BUCKET)
- except AirflowException:
- self.hook = None
- except oss2.exceptions.ServerError as e:
- if e.status == 403:
- self.hook = None
-
- @skip_test_if_no_valid_conn_id
- def test_init(self):
- assert self.hook.oss_conn_id == TEST_CONN_ID
-
- @skip_test_if_no_valid_conn_id
- def test_get_conn(self):
- assert self.hook.get_conn() is not None
-
- @skip_test_if_no_valid_conn_id
+ with mock.patch(
+ OSS_STRING.format('OSSHook.__init__'),
+ new=mock_oss_hook_default_project_id,
+ ):
+ self.hook = OSSHook(oss_conn_id=MOCK_OSS_CONN_ID)
+
def test_parse_oss_url(self):
- parsed = self.hook.parse_oss_url(f"oss://{TEST_BUCKET}/this/is/not/a-real-key.txt")
+ parsed = self.hook.parse_oss_url(f"oss://{MOCK_BUCKET_NAME}/this/is/not/a-real-key.txt")
print(parsed)
- assert parsed == (TEST_BUCKET, "this/is/not/a-real-key.txt"), "Incorrect parsing of the oss url"
+ assert parsed == (MOCK_BUCKET_NAME, "this/is/not/a-real-key.txt"), "Incorrect parsing of the oss url"
- @skip_test_if_no_valid_conn_id
def test_parse_oss_object_directory(self):
- parsed = self.hook.parse_oss_url(f"oss://{TEST_BUCKET}/this/is/not/a-real-oss-directory/")
+ parsed = self.hook.parse_oss_url(f"oss://{MOCK_BUCKET_NAME}/this/is/not/a-real-oss-directory/")
assert parsed == (
- TEST_BUCKET,
+ MOCK_BUCKET_NAME,
"this/is/not/a-real-oss-directory/",
), "Incorrect parsing of the oss url"
- @skip_test_if_no_valid_conn_id
- def test_get_bucket(self):
- assert self.hook.get_bucket(TEST_BUCKET) is not None
+ @mock.patch(OSS_STRING.format('oss2'))
+ def test_get_credential(self, mock_oss2):
+ self.hook.get_credential()
+ mock_oss2.Auth.assert_called_once_with('mock_access_key_id', 'mock_access_key_secret')
+
+ @mock.patch(OSS_STRING.format('OSSHook.get_credential'))
+ @mock.patch(OSS_STRING.format('oss2'))
+ def test_get_bucket(self, mock_oss2, mock_get_credential):
+ self.hook.get_bucket('mock_bucket_name')
+ mock_get_credential.assert_called_once_with()
+ mock_oss2.Bucket.assert_called_once_with(
+ mock_get_credential.return_value, 'http://oss-mock_region.aliyuncs.com', MOCK_BUCKET_NAME
+ )
+
+ @mock.patch(OSS_STRING.format('OSSHook.get_bucket'))
+ def test_object_exist(self, mock_service):
+ # Given
+ mock_bucket = mock_service.return_value
+ exists_method = mock_bucket.object_exists
+ exists_method.return_value = True
+
+ # When
+ res = self.hook.object_exists(MOCK_KEY, MOCK_BUCKET_NAME)
+
+ # Then
+ assert res is True
+ mock_service.assert_called_once_with(MOCK_BUCKET_NAME)
+ exists_method.assert_called_once_with(MOCK_KEY)
+
+ @mock.patch(OSS_STRING.format('OSSHook.get_bucket'))
+ def test_load_string(self, mock_service):
+ self.hook.load_string(MOCK_KEY, MOCK_CONTENT, MOCK_BUCKET_NAME)
+ mock_service.assert_called_once_with(MOCK_BUCKET_NAME)
+ mock_service.return_value.put_object.assert_called_once_with(MOCK_KEY, MOCK_CONTENT)
+
+ @mock.patch(OSS_STRING.format('OSSHook.get_bucket'))
+ def test_upload_local_file(self, mock_service):
+ self.hook.upload_local_file(MOCK_KEY, MOCK_FILE_PATH, MOCK_BUCKET_NAME)
+ mock_service.assert_called_once_with(MOCK_BUCKET_NAME)
+ mock_service.return_value.put_object_from_file.assert_called_once_with(MOCK_KEY, MOCK_FILE_PATH)
+
+ @mock.patch(OSS_STRING.format('OSSHook.get_bucket'))
+ def test_download_file(self, mock_service):
+ self.hook.download_file(MOCK_KEY, MOCK_FILE_PATH, MOCK_BUCKET_NAME)
+ mock_service.assert_called_once_with(MOCK_BUCKET_NAME)
+ mock_service.return_value.get_object_to_file(MOCK_KEY, MOCK_FILE_PATH)
+
+ @mock.patch(OSS_STRING.format('OSSHook.get_bucket'))
+ def test_delete_object(self, mock_service):
+ self.hook.delete_object(MOCK_KEY, MOCK_BUCKET_NAME)
+ mock_service.assert_called_once_with(MOCK_BUCKET_NAME)
+ mock_service.return_value.delete_object(MOCK_KEY)
+
+ @mock.patch(OSS_STRING.format('OSSHook.get_bucket'))
+ def test_delete_objects(self, mock_service):
+ self.hook.delete_objects(MOCK_KEYS, MOCK_BUCKET_NAME)
+ mock_service.assert_called_once_with(MOCK_BUCKET_NAME)
+ mock_service.return_value.batch_delete_objects(MOCK_KEYS)
+
+ @mock.patch(OSS_STRING.format('OSSHook.get_bucket'))
+ def test_delete_bucket(self, mock_service):
+ self.hook.delete_bucket(MOCK_BUCKET_NAME)
+ mock_service.assert_called_once_with(MOCK_BUCKET_NAME)
+ mock_service.return_value.delete_bucket.assert_called_once_with()
+
+ @mock.patch(OSS_STRING.format('OSSHook.get_bucket'))
+ def test_create_bucket(self, mock_service):
+ self.hook.create_bucket(MOCK_BUCKET_NAME)
+ mock_service.assert_called_once_with(MOCK_BUCKET_NAME)
+ mock_service.return_value.create_bucket.assert_called_once_with()
diff --git a/tests/providers/alibaba/cloud/utils/oss_mock.py b/tests/providers/alibaba/cloud/utils/oss_mock.py
new file mode 100644
index 0000000..7bbef54
--- /dev/null
+++ b/tests/providers/alibaba/cloud/utils/oss_mock.py
@@ -0,0 +1,36 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import json
+
+from airflow.models import Connection
+
+OSS_PROJECT_ID_HOOK_UNIT_TEST = 'example-project'
+
+
+def mock_oss_hook_default_project_id(self, oss_conn_id='mock_oss_default', region='mock_region'):
+ self.oss_conn_id = oss_conn_id
+ self.oss_conn = Connection(
+ extra=json.dumps(
+ {
+ 'auth_type': 'AK',
+ 'access_key_id': 'mock_access_key_id',
+ 'access_key_secret': 'mock_access_key_secret',
+ }
+ )
+ )
+ self.region = region