You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by as...@apache.org on 2021/08/11 15:12:36 UTC
[airflow] branch revert-17201-support_alibabacloud created (now
e64e5be)
This is an automated email from the ASF dual-hosted git repository.
ash pushed a change to branch revert-17201-support_alibabacloud
in repository https://gitbox.apache.org/repos/asf/airflow.git.
at e64e5be Revert "[AIRFLOW-17200] Add Alibaba Cloud OSS support (#17201)"
This branch includes the following new commits:
new e64e5be Revert "[AIRFLOW-17200] Add Alibaba Cloud OSS support (#17201)"
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[airflow] 01/01: Revert "[AIRFLOW-17200] Add Alibaba Cloud OSS
support (#17201)"
Posted by as...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ash pushed a commit to branch revert-17201-support_alibabacloud
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit e64e5bec2bb6fdbff64d3e943187b177df0ca50b
Author: Ash Berlin-Taylor <as...@firemirror.com>
AuthorDate: Wed Aug 11 16:12:08 2021 +0100
Revert "[AIRFLOW-17200] Add Alibaba Cloud OSS support (#17201)"
This reverts commit 75ca6543da3754f0dfb53d059588ad66a2f8235a.
---
CONTRIBUTING.rst | 2 +-
INSTALL | 2 +-
airflow/provider.yaml.schema.json | 1 -
airflow/providers/alibaba/CHANGELOG.rst | 25 --
airflow/providers/alibaba/__init__.py | 17 --
airflow/providers/alibaba/cloud/__init__.py | 17 --
.../cloud/example_dags/example_oss_bucket.py | 39 ---
.../cloud/example_dags/example_oss_object.py | 68 -----
airflow/providers/alibaba/cloud/hooks/__init__.py | 17 --
airflow/providers/alibaba/cloud/hooks/oss.py | 307 ---------------------
.../providers/alibaba/cloud/operators/__init__.py | 16 --
airflow/providers/alibaba/cloud/operators/oss.py | 221 ---------------
.../providers/alibaba/cloud/sensors/__init__.py | 16 --
airflow/providers/alibaba/cloud/sensors/oss_key.py | 93 -------
airflow/providers/alibaba/provider.yaml | 54 ----
airflow/utils/db.py | 12 -
docs/apache-airflow-providers-alibaba/commits.rst | 27 --
.../connections/alibaba.rst | 55 ----
docs/apache-airflow-providers-alibaba/index.rst | 74 -----
.../operators/index.rst | 26 --
.../operators/oss.rst | 55 ----
.../apache-airflow-providers-alibaba/redirects.txt | 1 -
docs/apache-airflow/extra-packages-ref.rst | 2 -
docs/conf.py | 1 -
.../alibaba/cloud/alibabacloud-oss.png | Bin 4968 -> 0 bytes
docs/spelling_wordlist.txt | 4 -
setup.py | 4 -
tests/providers/alibaba/__init__.py | 16 --
tests/providers/alibaba/cloud/__init__.py | 16 --
tests/providers/alibaba/cloud/hooks/__init__.py | 16 --
tests/providers/alibaba/cloud/hooks/test_oss.py | 68 -----
.../providers/alibaba/cloud/operators/__init__.py | 16 --
.../providers/alibaba/cloud/operators/test_oss.py | 126 ---------
tests/providers/alibaba/cloud/sensors/__init__.py | 16 --
.../alibaba/cloud/sensors/test_oss_key.py | 93 -------
tests/providers/alibaba/cloud/utils/__init__.py | 16 --
tests/providers/alibaba/cloud/utils/test_utils.py | 41 ---
37 files changed, 2 insertions(+), 1578 deletions(-)
diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst
index d8e2c43..773e0b9 100644
--- a/CONTRIBUTING.rst
+++ b/CONTRIBUTING.rst
@@ -585,7 +585,7 @@ This is the full list of those extras:
.. START EXTRAS HERE
-airbyte, alibaba, all, all_dbs, amazon, apache.atlas, apache.beam, apache.cassandra, apache.drill,
+airbyte, all, all_dbs, amazon, apache.atlas, apache.beam, apache.cassandra, apache.drill,
apache.druid, apache.hdfs, apache.hive, apache.kylin, apache.livy, apache.pig, apache.pinot,
apache.spark, apache.sqoop, apache.webhdfs, asana, async, atlas, aws, azure, cassandra, celery,
cgroups, cloudant, cncf.kubernetes, crypto, dask, databricks, datadog, deprecated_api, devel,
diff --git a/INSTALL b/INSTALL
index 47f48c3..c9af63f 100644
--- a/INSTALL
+++ b/INSTALL
@@ -89,7 +89,7 @@ The list of available extras:
# START EXTRAS HERE
-airbyte, alibaba, all, all_dbs, amazon, apache.atlas, apache.beam, apache.cassandra, apache.drill,
+airbyte, all, all_dbs, amazon, apache.atlas, apache.beam, apache.cassandra, apache.drill,
apache.druid, apache.hdfs, apache.hive, apache.kylin, apache.livy, apache.pig, apache.pinot,
apache.spark, apache.sqoop, apache.webhdfs, asana, async, atlas, aws, azure, cassandra, celery,
cgroups, cloudant, cncf.kubernetes, crypto, dask, databricks, datadog, deprecated_api, devel,
diff --git a/airflow/provider.yaml.schema.json b/airflow/provider.yaml.schema.json
index 840c902..6891206 100644
--- a/airflow/provider.yaml.schema.json
+++ b/airflow/provider.yaml.schema.json
@@ -66,7 +66,6 @@
"items": {
"type": "string",
"enum": [
- "alibaba",
"apache",
"aws",
"azure",
diff --git a/airflow/providers/alibaba/CHANGELOG.rst b/airflow/providers/alibaba/CHANGELOG.rst
deleted file mode 100644
index cef7dda..0000000
--- a/airflow/providers/alibaba/CHANGELOG.rst
+++ /dev/null
@@ -1,25 +0,0 @@
- .. Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- .. http://www.apache.org/licenses/LICENSE-2.0
-
- .. Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
-
-Changelog
----------
-
-1.0.0
-.....
-
-Initial version of the provider.
diff --git a/airflow/providers/alibaba/__init__.py b/airflow/providers/alibaba/__init__.py
deleted file mode 100644
index 217e5db..0000000
--- a/airflow/providers/alibaba/__init__.py
+++ /dev/null
@@ -1,17 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
diff --git a/airflow/providers/alibaba/cloud/__init__.py b/airflow/providers/alibaba/cloud/__init__.py
deleted file mode 100644
index 217e5db..0000000
--- a/airflow/providers/alibaba/cloud/__init__.py
+++ /dev/null
@@ -1,17 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
diff --git a/airflow/providers/alibaba/cloud/example_dags/example_oss_bucket.py b/airflow/providers/alibaba/cloud/example_dags/example_oss_bucket.py
deleted file mode 100644
index 51a524e..0000000
--- a/airflow/providers/alibaba/cloud/example_dags/example_oss_bucket.py
+++ /dev/null
@@ -1,39 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-from airflow.models.dag import DAG
-from airflow.providers.alibaba.cloud.operators.oss import OSSCreateBucketOperator, OSSDeleteBucketOperator
-from airflow.utils.dates import days_ago
-
-with DAG(
- dag_id='oss_bucket_dag',
- start_date=days_ago(2),
- max_active_runs=1,
- tags=['example'],
-) as dag:
-
- # [START howto_operator_oss_bucket]
- create_bucket = OSSCreateBucketOperator(
- oss_conn_id='oss_default', region='your region', task_id='task1', bucket_name='your bucket'
- )
-
- delete_bucket = OSSDeleteBucketOperator(
- oss_conn_id='oss_default', region='your region', task_id='task2', bucket_name='your bucket'
- )
- # [END howto_operator_oss_bucket]
-
- create_bucket >> delete_bucket
diff --git a/airflow/providers/alibaba/cloud/example_dags/example_oss_object.py b/airflow/providers/alibaba/cloud/example_dags/example_oss_object.py
deleted file mode 100644
index 4b69015..0000000
--- a/airflow/providers/alibaba/cloud/example_dags/example_oss_object.py
+++ /dev/null
@@ -1,68 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-from airflow.models.dag import DAG
-from airflow.providers.alibaba.cloud.operators.oss import (
- OSSDeleteBatchObjectOperator,
- OSSDeleteObjectOperator,
- OSSDownloadObjectOperator,
- OSSUploadObjectOperator,
-)
-from airflow.utils.dates import days_ago
-
-with DAG(
- dag_id='oss_object_dag',
- start_date=days_ago(2),
- max_active_runs=1,
- tags=['example'],
-) as dag:
-
- create_object = OSSUploadObjectOperator(
- file='your local file',
- key='your oss key',
- oss_conn_id='oss_default',
- region='your region',
- task_id='task1',
- bucket_name='your bucket',
- )
-
- download_object = OSSDownloadObjectOperator(
- file='your local file',
- key='your oss key',
- oss_conn_id='oss_default',
- region='your region',
- task_id='task2',
- bucket_name='your bucket',
- )
-
- delete_object = OSSDeleteObjectOperator(
- key='your oss key',
- oss_conn_id='oss_default',
- region='your region',
- task_id='task3',
- bucket_name='your bucket',
- )
-
- delete_batch_object = OSSDeleteBatchObjectOperator(
- keys=['obj1', 'obj2', 'obj3'],
- oss_conn_id='oss_default',
- region='your region',
- task_id='task4',
- bucket_name='your bucket',
- )
-
- create_object >> download_object >> delete_object >> delete_batch_object
diff --git a/airflow/providers/alibaba/cloud/hooks/__init__.py b/airflow/providers/alibaba/cloud/hooks/__init__.py
deleted file mode 100644
index 217e5db..0000000
--- a/airflow/providers/alibaba/cloud/hooks/__init__.py
+++ /dev/null
@@ -1,17 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
diff --git a/airflow/providers/alibaba/cloud/hooks/oss.py b/airflow/providers/alibaba/cloud/hooks/oss.py
deleted file mode 100644
index c318857..0000000
--- a/airflow/providers/alibaba/cloud/hooks/oss.py
+++ /dev/null
@@ -1,307 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-from functools import wraps
-from inspect import signature
-from typing import TYPE_CHECKING, Callable, Optional, TypeVar, cast
-from urllib.parse import urlparse
-
-import oss2
-from oss2.exceptions import ClientError
-
-from airflow.exceptions import AirflowException
-from airflow.hooks.base import BaseHook
-
-if TYPE_CHECKING:
- from airflow.models.connection import Connection
-
-T = TypeVar("T", bound=Callable)
-
-
-def provide_bucket_name(func: T) -> T:
- """
- Function decorator that unifies bucket name and key taken from the key
- in case no bucket name and at least a key has been passed to the function.
- """
- function_signature = signature(func)
-
- @wraps(func)
- def wrapper(*args, **kwargs) -> T:
- bound_args = function_signature.bind(*args, **kwargs)
- self = args[0]
- if 'bucket_name' not in bound_args.arguments or bound_args.arguments['bucket_name'] is None:
- if self.oss_conn_id:
- connection = self.get_connection(self.oss_conn_id)
- if connection.schema:
- bound_args.arguments['bucket_name'] = connection.schema
-
- return func(*bound_args.args, **bound_args.kwargs)
-
- return cast(T, wrapper)
-
-
-def unify_bucket_name_and_key(func: T) -> T:
- """
- Function decorator that unifies bucket name and key taken from the key
- in case no bucket name and at least a key has been passed to the function.
- """
- function_signature = signature(func)
-
- @wraps(func)
- def wrapper(*args, **kwargs) -> T:
- bound_args = function_signature.bind(*args, **kwargs)
-
- def get_key() -> Optional[str]:
- if 'key' in bound_args.arguments:
- return 'key'
- raise ValueError('Missing key parameter!')
-
- key_name = get_key()
- if 'bucket_name' not in bound_args.arguments or bound_args.arguments['bucket_name'] is None:
- bound_args.arguments['bucket_name'], bound_args.arguments['key'] = OSSHook.parse_oss_url(
- bound_args.arguments[key_name]
- )
-
- return func(*bound_args.args, **bound_args.kwargs)
-
- return cast(T, wrapper)
-
-
-class OSSHook(BaseHook):
- """Interact with Alibaba Cloud OSS, using the oss2 library."""
-
- conn_name_attr = 'alibabacloud_conn_id'
- default_conn_name = 'oss_default'
- conn_type = 'oss'
- hook_name = 'OSS'
-
- def __init__(self, region, oss_conn_id='oss_default', *args, **kwargs) -> None:
- self.oss_conn_id = oss_conn_id
- self.oss_conn = self.get_connection(oss_conn_id)
- self.region = region
- super().__init__(*args, **kwargs)
-
- def get_conn(self) -> "Connection":
- """Returns connection for the hook."""
- return self.oss_conn
-
- @staticmethod
- def parse_oss_url(ossurl: str) -> tuple:
- """
- Parses the OSS Url into a bucket name and key.
-
- :param ossurl: The OSS Url to parse.
- :rtype ossurl: str
- :return: the parsed bucket name and key
- :rtype: tuple of str
- """
- parsed_url = urlparse(ossurl)
-
- if not parsed_url.netloc:
- raise AirflowException(f'Please provide a bucket_name instead of "{ossurl}"')
-
- bucket_name = parsed_url.netloc
- key = parsed_url.path.lstrip('/')
-
- return bucket_name, key
-
- @unify_bucket_name_and_key
- def object_exists(self, key: str, bucket_name: Optional[str] = None) -> bool:
- """
- Check if object exists.
-
- :param key: the path of the object
- :type key: str
- :param bucket_name: the name of the bucket
- :type bucket_name: str
- :return: True if it exists and False if not.
- :rtype: bool
- """
- try:
- return self.get_bucket(bucket_name).object_exists(key)
- except ClientError as e:
- self.log.error(e.message)
- return False
-
- @provide_bucket_name
- def get_bucket(self, bucket_name: Optional[str] = None) -> oss2.api.Bucket:
- """
- Returns a oss2.Bucket object
-
- :param bucket_name: the name of the bucket
- :type bucket_name: str
- :return: the bucket object to the bucket name.
- :rtype: oss2.api.Bucket
- """
- auth = self.get_credential()
- return oss2.Bucket(auth, 'http://oss-' + self.region + '.aliyuncs.com', bucket_name)
-
- @unify_bucket_name_and_key
- def load_string(self, key: str, content: str, bucket_name: Optional[str] = None) -> None:
- """
- Loads a string to OSS
-
- :param key: the path of the object
- :type key: str
- :param content: str to set as content for the key.
- :type content: str
- :param bucket_name: the name of the bucket
- :type bucket_name: str
- """
- try:
- self.get_bucket(bucket_name).put_object(key, content)
- except Exception as e:
- raise AirflowException(f"Errors: {e}")
-
- @unify_bucket_name_and_key
- def upload_local_file(
- self,
- key: str,
- file: str,
- bucket_name: Optional[str] = None,
- ) -> None:
- """
- Upload a local file to OSS
-
- :param key: the OSS path of the object
- :type key: str
- :param file: local file to upload.
- :type file: str
- :param bucket_name: the name of the bucket
- :type bucket_name: str
- """
- try:
- self.get_bucket(bucket_name).put_object_from_file(key, file)
- except Exception as e:
- raise AirflowException(f"Errors when upload file: {e}")
-
- @unify_bucket_name_and_key
- def download_file(
- self,
- key: str,
- local_file: str,
- bucket_name: Optional[str] = None,
- ) -> Optional[str]:
- """
- Download file from OSS
-
- :param key: key of the file-like object to download.
- :type key: str
- :param local_file: local path + file name to save.
- :type local_file: str
- :param bucket_name: the name of the bucket
- :type bucket_name: str
- :return: the file name.
- :rtype: str
- """
- try:
- self.get_bucket(bucket_name).get_object_to_file(key, local_file)
- except Exception as e:
- self.log.error(e)
- return None
- return local_file
-
- @unify_bucket_name_and_key
- def delete_object(
- self,
- key: str,
- bucket_name: Optional[str] = None,
- ) -> None:
- """
- Delete object from OSS
-
- :param key: key of the object to delete.
- :type key: str
- :param bucket_name: the name of the bucket
- :type bucket_name: str
- """
- try:
- self.get_bucket(bucket_name).delete_object(key)
- except Exception as e:
- self.log.error(e)
- raise AirflowException(f"Errors when deleting: {key}")
-
- @unify_bucket_name_and_key
- def delete_objects(
- self,
- key: list,
- bucket_name: Optional[str] = None,
- ) -> None:
- """
- Delete objects from OSS
-
- :param key: keys list of the objects to delete.
- :type key: list of str
- :param bucket_name: the name of the bucket
- :type bucket_name: str
- """
- try:
- self.get_bucket(bucket_name).batch_delete_objects(key)
- except Exception as e:
- self.log.error(e)
- raise AirflowException(f"Errors when deleting: {key}")
-
- @provide_bucket_name
- def delete_bucket(
- self,
- bucket_name: Optional[str] = None,
- ) -> None:
- """
- Delete bucket from OSS
-
- :param bucket_name: the name of the bucket
- :type bucket_name: str
- """
- try:
- self.get_bucket(bucket_name).delete_bucket()
- except Exception as e:
- self.log.error(e)
- raise AirflowException(f"Errors when deleting: {bucket_name}")
-
- @provide_bucket_name
- def create_bucket(
- self,
- bucket_name: Optional[str] = None,
- ) -> None:
- """
- Create bucket
-
- :param bucket_name: the name of the bucket
- :type bucket_name: str
- """
- try:
- self.get_bucket(bucket_name).create_bucket()
- except Exception as e:
- self.log.error(e)
- raise AirflowException(f"Errors when create bucket: {bucket_name}")
-
- def get_credential(self) -> oss2.auth.Auth:
- extra_config = self.oss_conn.extra_dejson
- auth_type = extra_config.get('auth_type', None)
- if not auth_type:
- raise Exception("No auth_type specified in extra_config. ")
-
- if auth_type == 'AK':
- oss_access_key_id = extra_config.get('access_key_id', None)
- oss_access_key_secret = extra_config.get('access_key_secret', None)
- if not oss_access_key_id:
- raise Exception("No access_key_id is specified for connection: " + self.oss_conn_id)
- if not oss_access_key_secret:
- raise Exception("No access_key_secret is specified for connection: " + self.oss_conn_id)
- return oss2.Auth(oss_access_key_id, oss_access_key_secret)
- else:
- raise Exception("Unsupported auth_type: " + auth_type)
diff --git a/airflow/providers/alibaba/cloud/operators/__init__.py b/airflow/providers/alibaba/cloud/operators/__init__.py
deleted file mode 100644
index 13a8339..0000000
--- a/airflow/providers/alibaba/cloud/operators/__init__.py
+++ /dev/null
@@ -1,16 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
diff --git a/airflow/providers/alibaba/cloud/operators/oss.py b/airflow/providers/alibaba/cloud/operators/oss.py
deleted file mode 100644
index 2e5d318..0000000
--- a/airflow/providers/alibaba/cloud/operators/oss.py
+++ /dev/null
@@ -1,221 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-"""This module contains Alibaba Cloud OSS operators."""
-from typing import Optional
-
-from airflow.models import BaseOperator
-from airflow.providers.alibaba.cloud.hooks.oss import OSSHook
-
-
-class OSSCreateBucketOperator(BaseOperator):
- """
- This operator creates an OSS bucket
-
- :param region: OSS region you want to create bucket
- :type region: str
- :param bucket_name: This is bucket name you want to create
- :type bucket_name: str
- :param oss_conn_id: The Airflow connection used for OSS credentials.
- :type oss_conn_id: Optional[str]
- """
-
- def __init__(
- self,
- region: str,
- bucket_name: Optional[str] = None,
- oss_conn_id: Optional[str] = 'oss_default',
- **kwargs,
- ) -> None:
- super().__init__(**kwargs)
- self.oss_conn_id = oss_conn_id
- self.region = region
- self.bucket_name = bucket_name
-
- def execute(self, context):
- oss_hook = OSSHook(oss_conn_id=self.oss_conn_id, region=self.region)
- oss_hook.create_bucket(bucket_name=self.bucket_name)
-
-
-class OSSDeleteBucketOperator(BaseOperator):
- """
- This operator to delete an OSS bucket
-
- :param region: OSS region you want to create bucket
- :type region: str
- :param bucket_name: This is bucket name you want to delete
- :type bucket_name: str
- :param oss_conn_id: The Airflow connection used for OSS credentials.
- :type oss_conn_id: Optional[str]
- """
-
- def __init__(
- self,
- region: str,
- bucket_name: Optional[str] = None,
- oss_conn_id: Optional[str] = 'oss_default',
- **kwargs,
- ) -> None:
- super().__init__(**kwargs)
- self.oss_conn_id = oss_conn_id
- self.region = region
- self.bucket_name = bucket_name
-
- def execute(self, context):
- oss_hook = OSSHook(oss_conn_id=self.oss_conn_id, region=self.region)
- oss_hook.delete_bucket(bucket_name=self.bucket_name)
-
-
-class OSSUploadObjectOperator(BaseOperator):
- """
- This operator to upload an file-like object
-
- :param key: the OSS path of the object
- :type key: str
- :param file: local file to upload.
- :type file: str
- :param region: OSS region you want to create bucket
- :type region: str
- :param bucket_name: This is bucket name you want to create
- :type bucket_name: str
- :param oss_conn_id: The Airflow connection used for OSS credentials.
- :type oss_conn_id: Optional[str]
- """
-
- def __init__(
- self,
- key: str,
- file: str,
- region: str,
- bucket_name: Optional[str] = None,
- oss_conn_id: Optional[str] = 'oss_default',
- **kwargs,
- ) -> None:
- super().__init__(**kwargs)
- self.key = key
- self.file = file
- self.oss_conn_id = oss_conn_id
- self.region = region
- self.bucket_name = bucket_name
-
- def execute(self, context):
- oss_hook = OSSHook(oss_conn_id=self.oss_conn_id, region=self.region)
- oss_hook.upload_local_file(bucket_name=self.bucket_name, key=self.key, file=self.file)
-
-
-class OSSDownloadObjectOperator(BaseOperator):
- """
- This operator to Download an OSS object
-
- :param key: key of the object to download.
- :type key: str
- :param local_file: local path + file name to save.
- :type local_file: str
- :param region: OSS region
- :type region: str
- :param bucket_name: OSS bucket name
- :type bucket_name: str
- :param oss_conn_id: The Airflow connection used for OSS credentials.
- :type oss_conn_id: Optional[str]
- """
-
- def __init__(
- self,
- key: str,
- file: str,
- region: str,
- bucket_name: Optional[str] = None,
- oss_conn_id: Optional[str] = 'oss_default',
- **kwargs,
- ) -> None:
- super().__init__(**kwargs)
- self.key = key
- self.file = file
- self.oss_conn_id = oss_conn_id
- self.region = region
- self.bucket_name = bucket_name
-
- def execute(self, context):
- oss_hook = OSSHook(oss_conn_id=self.oss_conn_id, region=self.region)
- oss_hook.download_file(bucket_name=self.bucket_name, key=self.key, local_file=self.file)
-
-
-class OSSDeleteBatchObjectOperator(BaseOperator):
- """
- This operator to delete OSS objects
-
- :param key: key list of the objects to delete.
- :type key: str
- :param region: OSS region
- :type region: str
- :param bucket_name: OSS bucket name
- :type bucket_name: str
- :param oss_conn_id: The Airflow connection used for OSS credentials.
- :type oss_conn_id: Optional[str]
- """
-
- def __init__(
- self,
- keys: list,
- region: str,
- bucket_name: Optional[str] = None,
- oss_conn_id: Optional[str] = 'oss_default',
- **kwargs,
- ) -> None:
- super().__init__(**kwargs)
- self.keys = keys
- self.oss_conn_id = oss_conn_id
- self.region = region
- self.bucket_name = bucket_name
-
- def execute(self, context):
- oss_hook = OSSHook(oss_conn_id=self.oss_conn_id, region=self.region)
- oss_hook.delete_objects(bucket_name=self.bucket_name, key=self.keys)
-
-
-class OSSDeleteObjectOperator(BaseOperator):
- """
- This operator to delete an OSS object
-
- :param key: key of the object to delete.
- :type key: str
- :param region: OSS region
- :type region: str
- :param bucket_name: OSS bucket name
- :type bucket_name: str
- :param oss_conn_id: The Airflow connection used for OSS credentials.
- :type oss_conn_id: Optional[str]
- """
-
- def __init__(
- self,
- key: str,
- region: str,
- bucket_name: Optional[str] = None,
- oss_conn_id: Optional[str] = 'oss_default',
- **kwargs,
- ) -> None:
- super().__init__(**kwargs)
- self.key = key
- self.oss_conn_id = oss_conn_id
- self.region = region
- self.bucket_name = bucket_name
-
- def execute(self, context):
- oss_hook = OSSHook(oss_conn_id=self.oss_conn_id, region=self.region)
- oss_hook.delete_object(bucket_name=self.bucket_name, key=self.key)
diff --git a/airflow/providers/alibaba/cloud/sensors/__init__.py b/airflow/providers/alibaba/cloud/sensors/__init__.py
deleted file mode 100644
index 13a8339..0000000
--- a/airflow/providers/alibaba/cloud/sensors/__init__.py
+++ /dev/null
@@ -1,16 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
diff --git a/airflow/providers/alibaba/cloud/sensors/oss_key.py b/airflow/providers/alibaba/cloud/sensors/oss_key.py
deleted file mode 100644
index 22bd8e9..0000000
--- a/airflow/providers/alibaba/cloud/sensors/oss_key.py
+++ /dev/null
@@ -1,93 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-try:
- from functools import cached_property
-except ImportError:
- from cached_property import cached_property
-from typing import Optional
-from urllib.parse import urlparse
-
-from airflow.exceptions import AirflowException
-from airflow.providers.alibaba.cloud.hooks.oss import OSSHook
-from airflow.sensors.base import BaseSensorOperator
-
-
-class OSSKeySensor(BaseSensorOperator):
- """
- Waits for a key (a file-like instance on OSS) to be present in a OSS bucket.
- OSS being a key/value it does not support folders. The path is just a key
- a resource.
-
- :param bucket_key: The key being waited on. Supports full oss:// style url
- or relative path from root level. When it's specified as a full oss://
- url, please leave bucket_name as `None`.
- :type bucket_key: str
- :param region: OSS region
- :type region: str
- :param bucket_name: OSS bucket name
- :type bucket_name: str
- :param oss_conn_id: The Airflow connection used for OSS credentials.
- :type oss_conn_id: Optional[str]
- """
-
- template_fields = ('bucket_key', 'bucket_name')
-
- def __init__(
- self,
- bucket_key: str,
- region: str,
- bucket_name: Optional[str] = None,
- oss_conn_id: Optional[str] = 'oss_default',
- **kwargs,
- ):
- super().__init__(**kwargs)
-
- self.bucket_name = bucket_name
- self.bucket_key = bucket_key
- self.region = region
- self.oss_conn_id = oss_conn_id
- self.hook: Optional[OSSHook] = None
-
- def poke(self, context):
-
- if self.bucket_name is None:
- parsed_url = urlparse(self.bucket_key)
- if parsed_url.netloc == '':
- raise AirflowException('If key is a relative path from root, please provide a bucket_name')
- self.bucket_name = parsed_url.netloc
- self.bucket_key = parsed_url.path.lstrip('/')
- else:
- parsed_url = urlparse(self.bucket_key)
- if parsed_url.scheme != '' or parsed_url.netloc != '':
- raise AirflowException(
- 'If bucket_name is provided, bucket_key'
- + ' should be relative path from root'
- + ' level, rather than a full oss:// url'
- )
-
- self.log.info('Poking for key : oss://%s/%s', self.bucket_name, self.bucket_key)
- return self.get_hook.object_exists(key=self.bucket_key, bucket_name=self.bucket_name)
-
- @cached_property
- def get_hook(self) -> OSSHook:
- """Create and return an OSSHook"""
- if self.hook:
- return self.hook
-
- self.hook = OSSHook(oss_conn_id=self.oss_conn_id, region=self.region)
- return self.hook
diff --git a/airflow/providers/alibaba/provider.yaml b/airflow/providers/alibaba/provider.yaml
deleted file mode 100644
index f93b545..0000000
--- a/airflow/providers/alibaba/provider.yaml
+++ /dev/null
@@ -1,54 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
----
-package-name: apache-airflow-providers-alibaba
-name: Alibaba
-description: |
- Alibaba Cloud integration (including `Alibaba Cloud <https://www.alibabacloud.com//>`__).
-
-versions:
- - 1.0.0
-
-additional-dependencies:
- - apache-airflow>=2.1.0
-
-integrations:
- - integration-name: Alibaba Cloud OSS
- external-doc-url: https://www.alibabacloud.com/help/product/31815.htm
- logo: /integration-logos/alibaba/cloud/alibabacloud-oss.png
- how-to-guide:
- - /docs/apache-airflow-providers-alibaba/operators/oss.rst
- tags: [alibaba]
-
-operators:
- - integration-name: Alibaba Cloud OSS
- python-modules:
- - airflow.providers.alibaba.cloud.operators.oss
-
-sensors:
- - integration-name: Alibaba Cloud OSS
- python-modules:
- - airflow.providers.alibaba.cloud.sensors.oss_key
-
-hooks:
- - integration-name: Alibaba Cloud OSS
- python-modules:
- - airflow.providers.alibaba.cloud.hooks.oss
-
-hook-class-names:
- - airflow.providers.alibaba.cloud.hooks.oss.OSSHook
diff --git a/airflow/utils/db.py b/airflow/utils/db.py
index 5777aa8..644fb00 100644
--- a/airflow/utils/db.py
+++ b/airflow/utils/db.py
@@ -399,18 +399,6 @@ def create_default_connections(session=None):
)
merge_conn(
Connection(
- conn_id="oss_default",
- conn_type="oss",
- extra='''{
- "auth_type": "AK",
- "access_key_id": "<ACCESS_KEY_ID>",
- "access_key_secret": "<ACCESS_KEY_SECRET>"}
- ''',
- ),
- session,
- )
- merge_conn(
- Connection(
conn_id="pig_cli_default",
conn_type="pig_cli",
schema="default",
diff --git a/docs/apache-airflow-providers-alibaba/commits.rst b/docs/apache-airflow-providers-alibaba/commits.rst
deleted file mode 100644
index 7d5e85b..0000000
--- a/docs/apache-airflow-providers-alibaba/commits.rst
+++ /dev/null
@@ -1,27 +0,0 @@
-
- .. Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- .. http://www.apache.org/licenses/LICENSE-2.0
-
- .. Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
-
-Package apache-airflow-providers-alibaba
-----------------------------------------
-
-Alibaba integration (including `Alibabacloud <https://www.alibabacloud.com//>`__).
-
-
-This is detailed commit list of changes for versions provider package: ``alibaba.cloud``.
-For high-level changelog, see :doc:`package information including changelog <index>`.
diff --git a/docs/apache-airflow-providers-alibaba/connections/alibaba.rst b/docs/apache-airflow-providers-alibaba/connections/alibaba.rst
deleted file mode 100644
index d697b4d..0000000
--- a/docs/apache-airflow-providers-alibaba/connections/alibaba.rst
+++ /dev/null
@@ -1,55 +0,0 @@
- .. Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- .. http://www.apache.org/licenses/LICENSE-2.0
-
- .. Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
-Alibaba Cloud Connection
-========================
-
-Authenticating to Alibaba Cloud
--------------------------------
-
-Authentication may be performed using `Security Token Service (STS) or a signed URL <https://www.alibabacloud.com/help/doc-detail/32033.htm>`_ .
-
-Default Connection IDs
-----------------------
-
-The default connection ID is ``oss_default``.
-
-Configuring the Connection
---------------------------
-
-Schema (optional)
- Specify the default bucket name used for OSS hook.
-
-Extra (optional)
- Specify the extra parameters (as json dictionary) that can be used in Alibaba Cloud
- connection. The following parameters are all optional:
-
- * ``auth_type``: Auth type used to access Alibaba Cloud resource. Only support 'AK' now.
-
- * ``access_key_id``: Access key ID for Alibaba Cloud user.
- * ``access_key_secret``: Access key secret for Alibaba Cloud user.
-
-Examples for the **Extra** field
---------------------------------
-
-.. code-block:: json
-
- {
- "auth_type": "AK",
- "access_key_id": "",
- "access_key_secret": ""
- }
diff --git a/docs/apache-airflow-providers-alibaba/index.rst b/docs/apache-airflow-providers-alibaba/index.rst
deleted file mode 100644
index 89380a6..0000000
--- a/docs/apache-airflow-providers-alibaba/index.rst
+++ /dev/null
@@ -1,74 +0,0 @@
- .. Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- .. http://www.apache.org/licenses/LICENSE-2.0
-
- .. Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
-``apache-airflow-providers-alibaba``
-====================================
-
-Content
--------
-
-.. toctree::
- :maxdepth: 1
- :caption: Guides
-
- Connection types <connections/alibaba>
- Operators <operators/index>
-
-.. toctree::
- :maxdepth: 1
- :caption: References
-
- Python API <_api/airflow/providers/alibaba/index>
-
-.. toctree::
- :maxdepth: 1
- :caption: Resources
-
- Example DAGs <https://github.com/apache/airflow/tree/main/airflow/providers/alibaba/cloud/example_dags>
- PyPI Repository <https://pypi.org/project/apache-airflow-providers-alibaba/>
-
-.. THE REMINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN AT RELEASE TIME!
-
-
-.. toctree::
- :maxdepth: 1
- :caption: Commits
-
- Detailed list of commits <commits>
-
-
-Provider package
-----------------
-
-This is a provider package for ``alibaba.cloud`` provider. All classes for this provider package
-are in ``airflow.providers.alibaba.cloud`` python package.
-
-Installation
-------------
-
-You can install this package on top of an existing airflow 2.1+ installation via
-``pip install apache-airflow-providers-alibaba``
-
-PIP requirements
-----------------
-
-================== ====================
-PIP package Version required
-================== ====================
-``apache-airflow`` ``>=2.1.0``
-``oss2`` ``>=2.14.0``
-================== ====================
diff --git a/docs/apache-airflow-providers-alibaba/operators/index.rst b/docs/apache-airflow-providers-alibaba/operators/index.rst
deleted file mode 100644
index ccf9211..0000000
--- a/docs/apache-airflow-providers-alibaba/operators/index.rst
+++ /dev/null
@@ -1,26 +0,0 @@
- .. Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- .. http://www.apache.org/licenses/LICENSE-2.0
-
- .. Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
-Alibaba Cloud Operators
-=======================
-
-
-.. toctree::
- :maxdepth: 1
- :glob:
-
- *
diff --git a/docs/apache-airflow-providers-alibaba/operators/oss.rst b/docs/apache-airflow-providers-alibaba/operators/oss.rst
deleted file mode 100644
index 661fc4b..0000000
--- a/docs/apache-airflow-providers-alibaba/operators/oss.rst
+++ /dev/null
@@ -1,55 +0,0 @@
- .. Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- .. http://www.apache.org/licenses/LICENSE-2.0
-
- .. Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
-Alibaba Cloud OSS Operators
-===========================
-
-.. contents::
- :depth: 1
- :local:
-
-Overview
---------
-
-Airflow to Alibaba Cloud Object Storage Service (OSS) integration provides several operators to create and interact with OSS buckets.
-
- - :class:`~airflow.providers.alibaba.cloud.sensors.oss_key.OSSKeySensor`
- - :class:`~airflow.providers.alibaba.cloud.operators.oss.OSSCreateBucketOperator`
- - :class:`~airflow.providers.alibaba.cloud.operators.oss.OSSDeleteBucketOperator`
- - :class:`~airflow.providers.alibaba.cloud.operators.oss.OSSUploadObjectOperator`
- - :class:`~airflow.providers.alibaba.cloud.operators.oss.OSSDownloadObjectOperator`
- - :class:`~airflow.providers.alibaba.cloud.operators.oss.OSSDeleteBatchObjectOperator`
- - :class:`~airflow.providers.alibaba.cloud.operators.oss.OSSDeleteObjectOperator`
-
-Create and Delete Alibaba Cloud OSS Buckets
--------------------------------------------
-
-Purpose
-"""""""
-
-This example dag uses ``OSSCreateBucketOperator`` and ``OSSDeleteBucketOperator`` to create a
-new OSS bucket with a given bucket name then delete it.
-
-Defining tasks
-""""""""""""""
-
-In the following code we create a new bucket and then delete the bucket.
-
-.. exampleinclude:: /../../airflow/providers/alibaba/cloud/example_dags/example_oss_bucket.py
- :language: python
- :start-after: [START howto_operator_oss_bucket]
- :end-before: [END howto_operator_oss_bucket]
diff --git a/docs/apache-airflow-providers-alibaba/redirects.txt b/docs/apache-airflow-providers-alibaba/redirects.txt
deleted file mode 100644
index b6b18f5..0000000
--- a/docs/apache-airflow-providers-alibaba/redirects.txt
+++ /dev/null
@@ -1 +0,0 @@
-connections/index.rst connections/alibaba.rst
diff --git a/docs/apache-airflow/extra-packages-ref.rst b/docs/apache-airflow/extra-packages-ref.rst
index ae88728..29b6634 100644
--- a/docs/apache-airflow/extra-packages-ref.rst
+++ b/docs/apache-airflow/extra-packages-ref.rst
@@ -150,8 +150,6 @@ Those are extras that add dependencies needed for integration with external serv
+=====================+=====================================================+=====================================================+
| airbyte | ``pip install 'apache-airflow[airbyte]'`` | Airbyte hooks and operators |
+---------------------+-----------------------------------------------------+-----------------------------------------------------+
-| alibaba | ``pip install 'apache-airflow[alibaba]'`` | Alibaba Cloud |
-+---------------------+-----------------------------------------------------+-----------------------------------------------------+
| amazon | ``pip install 'apache-airflow[amazon]'`` | Amazon Web Services |
+---------------------+-----------------------------------------------------+-----------------------------------------------------+
| asana | ``pip install 'apache-airflow[asana]'`` | Asana hooks and operators |
diff --git a/docs/conf.py b/docs/conf.py
index af642b8..dca3246 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -500,7 +500,6 @@ autodoc_mock_imports = [
'jira',
'kubernetes',
'msrestazure',
- 'oss2',
'pandas',
'pandas_gbq',
'paramiko',
diff --git a/docs/integration-logos/alibaba/cloud/alibabacloud-oss.png b/docs/integration-logos/alibaba/cloud/alibabacloud-oss.png
deleted file mode 100644
index abe40cd..0000000
Binary files a/docs/integration-logos/alibaba/cloud/alibabacloud-oss.png and /dev/null differ
diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt
index 47c82ba..d240ac9 100644
--- a/docs/spelling_wordlist.txt
+++ b/docs/spelling_wordlist.txt
@@ -4,7 +4,6 @@ AgentKey
Airbnb
Airbyte
AirflowException
-Alibaba
Alphasort
Analytics
AnalyzeEntitiesResponse
@@ -242,7 +241,6 @@ Nones
NotFound
Nullable
OAuth
-OSS
Oauth
Oauthlib
Okta
@@ -402,7 +400,6 @@ airbyte
airflowignore
ajax
alertPolicies
-alibaba
allAuthenticatedUsers
allUsers
allowinsert
@@ -1009,7 +1006,6 @@ orchestrator
orgtbl
orm
os
-oss
ot
overridable
oversubscription
diff --git a/setup.py b/setup.py
index 2eaaa47..9d8597c 100644
--- a/setup.py
+++ b/setup.py
@@ -179,9 +179,6 @@ def write_version(filename: str = os.path.join(*[my_dir, "airflow", "git_version
# 'Start dependencies group' and 'Start dependencies group' are mark for ./scripts/ci/check_order_setup.py
# If you change this mark you should also change ./scripts/ci/check_order_setup.py
# Start dependencies group
-alibaba = [
- 'oss2>=2.14.0',
-]
amazon = [
'boto3>=1.15.0,<1.18.0',
'watchtower~=1.0.6',
@@ -540,7 +537,6 @@ devel_hadoop = devel_minreq + hdfs + hive + kerberos + presto + webhdfs
# Dict of all providers which are part of the Apache Airflow repository together with their requirements
PROVIDERS_REQUIREMENTS: Dict[str, List[str]] = {
'airbyte': http_provider,
- 'alibaba': alibaba,
'amazon': amazon,
'apache.beam': apache_beam,
'apache.cassandra': cassandra,
diff --git a/tests/providers/alibaba/__init__.py b/tests/providers/alibaba/__init__.py
deleted file mode 100644
index 13a8339..0000000
--- a/tests/providers/alibaba/__init__.py
+++ /dev/null
@@ -1,16 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
diff --git a/tests/providers/alibaba/cloud/__init__.py b/tests/providers/alibaba/cloud/__init__.py
deleted file mode 100644
index 13a8339..0000000
--- a/tests/providers/alibaba/cloud/__init__.py
+++ /dev/null
@@ -1,16 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
diff --git a/tests/providers/alibaba/cloud/hooks/__init__.py b/tests/providers/alibaba/cloud/hooks/__init__.py
deleted file mode 100644
index 13a8339..0000000
--- a/tests/providers/alibaba/cloud/hooks/__init__.py
+++ /dev/null
@@ -1,16 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
diff --git a/tests/providers/alibaba/cloud/hooks/test_oss.py b/tests/providers/alibaba/cloud/hooks/test_oss.py
deleted file mode 100644
index b5e7d0b..0000000
--- a/tests/providers/alibaba/cloud/hooks/test_oss.py
+++ /dev/null
@@ -1,68 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-import os
-import unittest
-
-import oss2
-
-from airflow.exceptions import AirflowException
-from airflow.providers.alibaba.cloud.hooks.oss import OSSHook
-from tests.providers.alibaba.cloud.utils.test_utils import skip_test_if_no_valid_conn_id
-
-TEST_CONN_ID = os.environ.get('TEST_OSS_CONN_ID', 'oss_default')
-TEST_REGION = os.environ.get('TEST_OSS_REGION', 'cn-hangzhou')
-TEST_BUCKET = os.environ.get('TEST_OSS_BUCKET', 'test-bucket')
-
-
-class TestOSSHook(unittest.TestCase):
- def setUp(self):
- try:
- self.hook = OSSHook(region=TEST_REGION, oss_conn_id=TEST_CONN_ID)
- self.hook.object_exists(key='test-obj', bucket_name=TEST_BUCKET)
- except AirflowException:
- self.hook = None
- except oss2.exceptions.ServerError as e:
- if e.status == 403:
- self.hook = None
-
- @skip_test_if_no_valid_conn_id
- def test_init(self):
- assert self.hook.oss_conn_id == TEST_CONN_ID
-
- @skip_test_if_no_valid_conn_id
- def test_get_conn(self):
- assert self.hook.get_conn() is not None
-
- @skip_test_if_no_valid_conn_id
- def test_parse_oss_url(self):
- parsed = self.hook.parse_oss_url(f"oss://{TEST_BUCKET}/this/is/not/a-real-key.txt")
- print(parsed)
- assert parsed == (TEST_BUCKET, "this/is/not/a-real-key.txt"), "Incorrect parsing of the oss url"
-
- @skip_test_if_no_valid_conn_id
- def test_parse_oss_object_directory(self):
- parsed = self.hook.parse_oss_url(f"oss://{TEST_BUCKET}/this/is/not/a-real-oss-directory/")
- assert parsed == (
- TEST_BUCKET,
- "this/is/not/a-real-oss-directory/",
- ), "Incorrect parsing of the oss url"
-
- @skip_test_if_no_valid_conn_id
- def test_get_bucket(self):
- assert self.hook.get_bucket(TEST_BUCKET) is not None
diff --git a/tests/providers/alibaba/cloud/operators/__init__.py b/tests/providers/alibaba/cloud/operators/__init__.py
deleted file mode 100644
index 13a8339..0000000
--- a/tests/providers/alibaba/cloud/operators/__init__.py
+++ /dev/null
@@ -1,16 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
diff --git a/tests/providers/alibaba/cloud/operators/test_oss.py b/tests/providers/alibaba/cloud/operators/test_oss.py
deleted file mode 100644
index c70aff3..0000000
--- a/tests/providers/alibaba/cloud/operators/test_oss.py
+++ /dev/null
@@ -1,126 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-import os
-import unittest
-
-import oss2
-
-from airflow.exceptions import AirflowException
-from airflow.providers.alibaba.cloud.hooks.oss import OSSHook
-from airflow.providers.alibaba.cloud.operators.oss import (
- OSSCreateBucketOperator,
- OSSDeleteBatchObjectOperator,
- OSSDeleteBucketOperator,
- OSSDeleteObjectOperator,
- OSSDownloadObjectOperator,
- OSSUploadObjectOperator,
-)
-from tests.providers.alibaba.cloud.utils.test_utils import skip_test_if_no_valid_conn_id
-
-TEST_CONN_ID = os.environ.get('TEST_OSS_CONN_ID', 'oss_default')
-TEST_REGION = os.environ.get('TEST_OSS_REGION', 'cn-hangzhou')
-TEST_BUCKET = os.environ.get('TEST_OSS_BUCKET', 'test-bucket')
-TEST_FILE_PATH = '/tmp/airflow-test'
-
-
-class TestOSSOperator(unittest.TestCase):
- def setUp(self):
- self.create_bucket_operator = OSSCreateBucketOperator(
- oss_conn_id=TEST_CONN_ID, region=TEST_REGION, bucket_name=TEST_BUCKET, task_id='task-1'
- )
- self.delete_bucket_operator = OSSDeleteBucketOperator(
- oss_conn_id=TEST_CONN_ID, region=TEST_REGION, bucket_name=TEST_BUCKET, task_id='task-2'
- )
- try:
- self.hook = OSSHook(region=TEST_REGION)
- self.hook.object_exists(key='test-obj', bucket_name=TEST_BUCKET)
- except AirflowException:
- self.hook = None
- except oss2.exceptions.ServerError as e:
- if e.status == 403:
- self.hook = None
-
- @skip_test_if_no_valid_conn_id
- def test_init(self):
- assert self.create_bucket_operator.oss_conn_id == TEST_CONN_ID
-
- @skip_test_if_no_valid_conn_id
- def test_create_delete_bucket(self):
- self.create_bucket_operator.execute({})
- self.delete_bucket_operator.execute({})
-
- @skip_test_if_no_valid_conn_id
- def test_object(self):
- self.create_bucket_operator.execute({})
-
- upload_file = f'{TEST_FILE_PATH}_upload_1'
- if not os.path.exists(upload_file):
- with open(upload_file, 'w') as f:
- f.write('test')
- upload_object_operator = OSSUploadObjectOperator(
- key='obj',
- file=upload_file,
- oss_conn_id=TEST_CONN_ID,
- region=TEST_REGION,
- bucket_name=TEST_BUCKET,
- task_id='task-1',
- )
- upload_object_operator.execute({})
- assert self.hook.object_exists(key='obj', bucket_name=TEST_BUCKET)
-
- download_file = f'{TEST_FILE_PATH}_download_1'
- download_object_operator = OSSDownloadObjectOperator(
- key='obj',
- file=download_file,
- oss_conn_id=TEST_CONN_ID,
- region=TEST_REGION,
- bucket_name=TEST_BUCKET,
- task_id='task-2',
- )
- download_object_operator.execute({})
- assert os.path.exists(download_file)
-
- delete_object_operator = OSSDeleteObjectOperator(
- key='obj', oss_conn_id=TEST_CONN_ID, region=TEST_REGION, bucket_name=TEST_BUCKET, task_id='task-3'
- )
- delete_object_operator.execute({})
- assert self.hook.object_exists(key='obj', bucket_name=TEST_BUCKET) is False
-
- upload_object_operator = OSSUploadObjectOperator(
- key='obj',
- file=upload_file,
- oss_conn_id=TEST_CONN_ID,
- region=TEST_REGION,
- bucket_name=TEST_BUCKET,
- task_id='task-4',
- )
- upload_object_operator.execute({})
- assert self.hook.object_exists(key='obj', bucket_name=TEST_BUCKET)
-
- delete_objects_operator = OSSDeleteBatchObjectOperator(
- keys=['obj'],
- oss_conn_id=TEST_CONN_ID,
- region=TEST_REGION,
- bucket_name=TEST_BUCKET,
- task_id='task-5',
- )
- delete_objects_operator.execute({})
- assert self.hook.object_exists(key='obj', bucket_name=TEST_BUCKET) is False
-
- self.delete_bucket_operator.execute({})
diff --git a/tests/providers/alibaba/cloud/sensors/__init__.py b/tests/providers/alibaba/cloud/sensors/__init__.py
deleted file mode 100644
index 13a8339..0000000
--- a/tests/providers/alibaba/cloud/sensors/__init__.py
+++ /dev/null
@@ -1,16 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
diff --git a/tests/providers/alibaba/cloud/sensors/test_oss_key.py b/tests/providers/alibaba/cloud/sensors/test_oss_key.py
deleted file mode 100644
index 19ed30c..0000000
--- a/tests/providers/alibaba/cloud/sensors/test_oss_key.py
+++ /dev/null
@@ -1,93 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-import os
-import unittest
-
-import oss2
-
-from airflow.exceptions import AirflowException
-from airflow.providers.alibaba.cloud.hooks.oss import OSSHook
-from airflow.providers.alibaba.cloud.operators.oss import (
- OSSCreateBucketOperator,
- OSSDeleteBucketOperator,
- OSSDeleteObjectOperator,
- OSSUploadObjectOperator,
-)
-from airflow.providers.alibaba.cloud.sensors.oss_key import OSSKeySensor
-from tests.providers.alibaba.cloud.utils.test_utils import skip_test_if_no_valid_conn_id
-
-TEST_CONN_ID = os.environ.get('TEST_OSS_CONN_ID', 'oss_default')
-TEST_REGION = os.environ.get('TEST_OSS_REGION', 'cn-hangzhou')
-TEST_BUCKET = os.environ.get('TEST_OSS_BUCKET', 'test-bucket')
-TEST_FILE_PATH = '/tmp/airflow-test'
-
-
-class TestOSSSensor(unittest.TestCase):
- def setUp(self):
- self.sensor = OSSKeySensor(
- bucket_key='obj',
- oss_conn_id=TEST_CONN_ID,
- region=TEST_REGION,
- bucket_name=TEST_BUCKET,
- task_id='task-1',
- )
- try:
- self.hook = OSSHook(region=TEST_REGION, oss_conn_id=TEST_CONN_ID)
- self.hook.object_exists(key='test-obj', bucket_name=TEST_BUCKET)
- except AirflowException:
- self.hook = None
- except oss2.exceptions.ServerError as e:
- if e.status == 403:
- self.hook = None
-
- @skip_test_if_no_valid_conn_id
- def test_init(self):
- assert self.sensor.oss_conn_id == TEST_CONN_ID
-
- @skip_test_if_no_valid_conn_id
- def test_poke(self):
- create_bucket_operator = OSSCreateBucketOperator(
- oss_conn_id=TEST_CONN_ID, region=TEST_REGION, bucket_name=TEST_BUCKET, task_id='task-2'
- )
- create_bucket_operator.execute({})
-
- upload_file = f'{TEST_FILE_PATH}_upload_1'
- if not os.path.exists(upload_file):
- with open(upload_file, 'w') as f:
- f.write('test')
- upload_object_operator = OSSUploadObjectOperator(
- key='obj',
- file=upload_file,
- oss_conn_id=TEST_CONN_ID,
- region=TEST_REGION,
- bucket_name=TEST_BUCKET,
- task_id='task-3',
- )
- upload_object_operator.execute({})
- assert self.sensor.poke({})
-
- delete_object_operator = OSSDeleteObjectOperator(
- key='obj', oss_conn_id=TEST_CONN_ID, region=TEST_REGION, bucket_name=TEST_BUCKET, task_id='task-4'
- )
- delete_object_operator.execute({})
-
- delete_bucket_operator = OSSDeleteBucketOperator(
- oss_conn_id=TEST_CONN_ID, region=TEST_REGION, bucket_name=TEST_BUCKET, task_id='task-5'
- )
- delete_bucket_operator.execute({})
diff --git a/tests/providers/alibaba/cloud/utils/__init__.py b/tests/providers/alibaba/cloud/utils/__init__.py
deleted file mode 100644
index 13a8339..0000000
--- a/tests/providers/alibaba/cloud/utils/__init__.py
+++ /dev/null
@@ -1,16 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
diff --git a/tests/providers/alibaba/cloud/utils/test_utils.py b/tests/providers/alibaba/cloud/utils/test_utils.py
deleted file mode 100644
index 82af3ce..0000000
--- a/tests/providers/alibaba/cloud/utils/test_utils.py
+++ /dev/null
@@ -1,41 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-from functools import wraps
-from inspect import signature
-from typing import Callable, TypeVar, cast
-
-T = TypeVar("T", bound=Callable)
-
-
-def skip_test_if_no_valid_conn_id(func: T) -> T:
- """
- Function decorator that skip this test function if no valid connection id is specified.
- """
- function_signature = signature(func)
-
- @wraps(func)
- def wrapper(*args, **kwargs) -> None:
- bound_args = function_signature.bind(*args, **kwargs)
- self = args[0]
-
- if self.hook is not None:
- return func(*bound_args.args, **bound_args.kwargs)
- else:
- return None
-
- return cast(T, wrapper)