You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2022/06/05 15:41:46 UTC
[airflow] branch main updated: Migrate Microsoft system tests to new design AIP-47 (#24225)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new d71787e0d7 Migrate Microsoft system tests to new design AIP-47 (#24225)
d71787e0d7 is described below
commit d71787e0d7423e8a116811e86edf76588b3c7017
Author: Chenglong Yan <al...@gmail.com>
AuthorDate: Sun Jun 5 23:41:37 2022 +0800
Migrate Microsoft system tests to new design AIP-47 (#24225)
closes: #22432
related: #22452
---
.../microsoft/azure/hooks/test_fileshare_system.py | 38 ---------------
.../azure/operators/test_adls_delete_system.py | 48 ------------------
.../azure/transfers/test_local_to_adls_system.py | 48 ------------------
.../azure/transfers/test_local_to_wasb_system.py | 48 ------------------
.../azure/transfers/test_sftp_to_wasb_system.py | 57 ----------------------
5 files changed, 239 deletions(-)
diff --git a/tests/providers/microsoft/azure/hooks/test_fileshare_system.py b/tests/providers/microsoft/azure/hooks/test_fileshare_system.py
deleted file mode 100644
index 451bd0dba0..0000000000
--- a/tests/providers/microsoft/azure/hooks/test_fileshare_system.py
+++ /dev/null
@@ -1,38 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-import os
-
-import pytest
-
-from tests.test_utils.azure_system_helpers import (
- AZURE_DAG_FOLDER,
- AzureSystemTest,
- provide_wasb_default_connection,
-)
-
-CREDENTIALS_DIR = os.environ.get('CREDENTIALS_DIR', '/files/airflow-breeze-config/keys')
-WASB_DEFAULT_KEY = 'wasb_key.json'
-CREDENTIALS_PATH = os.path.join(CREDENTIALS_DIR, WASB_DEFAULT_KEY)
-
-
-@pytest.mark.backend('postgres', 'mysql')
-@pytest.mark.credential_file(WASB_DEFAULT_KEY)
-class FileshareSystem(AzureSystemTest):
- @provide_wasb_default_connection(CREDENTIALS_PATH)
- def test_run_example_fileshare(self):
- self.run_dag('example_fileshare', AZURE_DAG_FOLDER)
diff --git a/tests/providers/microsoft/azure/operators/test_adls_delete_system.py b/tests/providers/microsoft/azure/operators/test_adls_delete_system.py
deleted file mode 100644
index 9a5f387d37..0000000000
--- a/tests/providers/microsoft/azure/operators/test_adls_delete_system.py
+++ /dev/null
@@ -1,48 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-import os
-
-import pytest
-
-from tests.system.providers.microsoft.azure.example_local_to_adls import LOCAL_FILE_PATH
-from tests.test_utils.azure_system_helpers import (
- AZURE_DAG_FOLDER,
- AzureSystemTest,
- provide_azure_data_lake_default_connection,
-)
-
-CREDENTIALS_DIR = os.environ.get('CREDENTIALS_DIR', '/files/airflow-breeze-config/keys')
-DATA_LAKE_DEFAULT_KEY = 'azure_data_lake.json'
-CREDENTIALS_PATH = os.path.join(CREDENTIALS_DIR, DATA_LAKE_DEFAULT_KEY)
-
-
-@pytest.mark.backend('postgres', 'mysql')
-@pytest.mark.credential_file(DATA_LAKE_DEFAULT_KEY)
-class ADLSDeleteSystem(AzureSystemTest):
- def setUp(self):
- super().setUp()
- with open(LOCAL_FILE_PATH, 'w+') as file:
- file.writelines(['example test files'])
-
- def tearDown(self):
- os.remove(LOCAL_FILE_PATH)
- super().tearDown()
-
- @provide_azure_data_lake_default_connection(CREDENTIALS_PATH)
- def test_run_example_adls_delete(self):
- self.run_dag('example_adls_delete', AZURE_DAG_FOLDER)
diff --git a/tests/providers/microsoft/azure/transfers/test_local_to_adls_system.py b/tests/providers/microsoft/azure/transfers/test_local_to_adls_system.py
deleted file mode 100644
index 99ed741067..0000000000
--- a/tests/providers/microsoft/azure/transfers/test_local_to_adls_system.py
+++ /dev/null
@@ -1,48 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-import os
-
-import pytest
-
-from tests.system.providers.microsoft.azure.example_local_to_adls import LOCAL_FILE_PATH
-from tests.test_utils.azure_system_helpers import (
- AZURE_DAG_FOLDER,
- AzureSystemTest,
- provide_azure_data_lake_default_connection,
-)
-
-CREDENTIALS_DIR = os.environ.get('CREDENTIALS_DIR', '/files/airflow-breeze-config/keys')
-DATA_LAKE_DEFAULT_KEY = 'azure_data_lake.json'
-CREDENTIALS_PATH = os.path.join(CREDENTIALS_DIR, DATA_LAKE_DEFAULT_KEY)
-
-
-@pytest.mark.backend('postgres', 'mysql')
-@pytest.mark.credential_file(DATA_LAKE_DEFAULT_KEY)
-class LocalToAdlsSystem(AzureSystemTest):
- def setUp(self):
- super().setUp()
- with open(LOCAL_FILE_PATH, 'w+') as file:
- file.writelines(['example test files'])
-
- def tearDown(self):
- os.remove(LOCAL_FILE_PATH)
- super().tearDown()
-
- @provide_azure_data_lake_default_connection(CREDENTIALS_PATH)
- def test_run_example_local_to_adls(self):
- self.run_dag('example_local_to_adls', AZURE_DAG_FOLDER)
diff --git a/tests/providers/microsoft/azure/transfers/test_local_to_wasb_system.py b/tests/providers/microsoft/azure/transfers/test_local_to_wasb_system.py
deleted file mode 100644
index f57ac8ca97..0000000000
--- a/tests/providers/microsoft/azure/transfers/test_local_to_wasb_system.py
+++ /dev/null
@@ -1,48 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-import os
-
-import pytest
-
-from tests.system.providers.microsoft.azure.example_local_to_wasb import PATH_TO_UPLOAD_FILE # type: ignore
-from tests.test_utils.azure_system_helpers import (
- AZURE_DAG_FOLDER,
- AzureSystemTest,
- provide_wasb_default_connection,
-)
-
-CREDENTIALS_DIR = os.environ.get('CREDENTIALS_DIR', '/files/airflow-breeze-config/keys')
-WASB_DEFAULT_KEY = 'wasb_key.json'
-CREDENTIALS_PATH = os.path.join(CREDENTIALS_DIR, WASB_DEFAULT_KEY)
-
-
-@pytest.mark.backend('postgres', 'mysql')
-@pytest.mark.credential_file(WASB_DEFAULT_KEY)
-class LocalToWasbSystem(AzureSystemTest):
- def setUp(self):
- super().setUp()
- with open(PATH_TO_UPLOAD_FILE, 'w+') as file:
- file.writelines(['example test files'])
-
- def tearDown(self):
- os.remove(PATH_TO_UPLOAD_FILE)
- super().tearDown()
-
- @provide_wasb_default_connection(CREDENTIALS_PATH)
- def test_run_example_local_to_wasb(self):
- self.run_dag('example_local_to_wasb', AZURE_DAG_FOLDER)
diff --git a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb_system.py b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb_system.py
deleted file mode 100644
index 91256539b2..0000000000
--- a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb_system.py
+++ /dev/null
@@ -1,57 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-
-import os
-
-import pytest
-
-from tests.system.providers.microsoft.azure.example_sftp_to_wasb import (
- FILE_COMPLETE_PATH,
- LOCAL_FILE_PATH,
- SAMPLE_FILENAME,
-)
-from tests.test_utils.azure_system_helpers import (
- AZURE_DAG_FOLDER,
- AzureSystemTest,
- provide_wasb_default_connection,
-)
-from tests.test_utils.sftp_system_helpers import provide_sftp_default_connection
-
-CREDENTIALS_DIR = os.environ.get('CREDENTIALS_DIR', '/files/airflow-breeze-config/keys')
-SFTP_DEFAULT_KEY = 'sftp_key.json'
-WASB_DEFAULT_KEY = 'wasb_key.json'
-CREDENTIALS_SFTP_PATH = os.path.join(CREDENTIALS_DIR, SFTP_DEFAULT_KEY)
-CREDENTIALS_WASB_PATH = os.path.join(CREDENTIALS_DIR, WASB_DEFAULT_KEY)
-
-
-@pytest.mark.backend('postgres', 'mysql')
-@pytest.mark.credential_file(WASB_DEFAULT_KEY)
-@pytest.mark.credential_file(SFTP_DEFAULT_KEY)
-class TestSFTPToWasbSystem(AzureSystemTest):
- def setUp(self):
- super().setUp()
- self.create_dummy_file(SAMPLE_FILENAME, LOCAL_FILE_PATH)
-
- def tearDown(self):
- os.remove(FILE_COMPLETE_PATH)
- super().tearDown()
-
- @provide_wasb_default_connection(CREDENTIALS_WASB_PATH)
- @provide_sftp_default_connection(CREDENTIALS_SFTP_PATH)
- def test_run_example_file_to_wasb(self):
- self.run_dag('example_sftp_to_wasb', AZURE_DAG_FOLDER)