You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2021/07/18 17:21:28 UTC
[airflow] branch main updated: Added docs & doc ref's for AWS
transfer operators between SFTP & S3 (#16964)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new cda7833 Added docs & doc ref's for AWS transfer operators between SFTP & S3 (#16964)
cda7833 is described below
commit cda78333b4ce9304abe315ab1afe41efe17fd2da
Author: pyerbiz <39...@users.noreply.github.com>
AuthorDate: Sun Jul 18 22:51:10 2021 +0530
Added docs & doc ref's for AWS transfer operators between SFTP & S3 (#16964)
---
.../amazon/aws/example_dags/example_s3_to_sftp.py | 43 ++++++++++++++++++++++
.../amazon/aws/example_dags/example_sftp_to_s3.py | 42 +++++++++++++++++++++
.../providers/amazon/aws/transfers/s3_to_sftp.py | 4 ++
.../providers/amazon/aws/transfers/sftp_to_s3.py | 4 ++
airflow/providers/amazon/provider.yaml | 2 +
.../operators/transfer/s3_to_sftp.rst | 42 +++++++++++++++++++++
.../operators/transfer/sftp_to_s3.rst | 40 ++++++++++++++++++++
7 files changed, 177 insertions(+)
diff --git a/airflow/providers/amazon/aws/example_dags/example_s3_to_sftp.py b/airflow/providers/amazon/aws/example_dags/example_s3_to_sftp.py
new file mode 100644
index 0000000..1326a36
--- /dev/null
+++ b/airflow/providers/amazon/aws/example_dags/example_s3_to_sftp.py
@@ -0,0 +1,43 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+
+import os
+
+from airflow import models
+from airflow.providers.amazon.aws.transfers.s3_to_sftp import S3ToSFTPOperator
+from airflow.utils.dates import days_ago
+
+S3_BUCKET = os.environ.get("S3_BUCKET", "test-bucket")
+S3_KEY = os.environ.get("S3_KEY", "key")
+
+with models.DAG(
+ "example_s3_to_sftp",
+ schedule_interval=None,
+ start_date=days_ago(1), # Override to match your needs
+) as dag:
+
+ # [START howto_s3_transfer_data_to_sftp]
+ create_s3_to_sftp_job = S3ToSFTPOperator(
+ task_id="create_to_s3_sftp_job",
+ sftp_conn_id="sftp_conn_id",
+ sftp_path="sftp_path",
+ s3_conn_id="s3_conn_id",
+ s3_bucket=S3_BUCKET,
+ s3_key=S3_KEY,
+ )
+ # [END howto_s3_transfer_data_to_sftp]
diff --git a/airflow/providers/amazon/aws/example_dags/example_sftp_to_s3.py b/airflow/providers/amazon/aws/example_dags/example_sftp_to_s3.py
new file mode 100644
index 0000000..f69879f
--- /dev/null
+++ b/airflow/providers/amazon/aws/example_dags/example_sftp_to_s3.py
@@ -0,0 +1,42 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+
+import os
+
+from airflow import models
+from airflow.providers.amazon.aws.transfers.sftp_to_s3 import SFTPToS3Operator
+from airflow.utils.dates import days_ago
+
+S3_BUCKET = os.environ.get("S3_BUCKET", "test-bucket")
+S3_KEY = os.environ.get("S3_KEY", "key")
+
+with models.DAG(
+ "example_sftp_to_s3",
+ schedule_interval=None,
+ start_date=days_ago(1), # Override to match your needs
+) as dag:
+ # [START howto_sftp_transfer_data_to_s3]
+ create_sftp_to_s3_job = SFTPToS3Operator(
+ task_id="create_sftp_to_s3_job",
+ sftp_conn_id="sftp_conn_id",
+ sftp_path="sftp_path",
+ s3_conn_id="s3_conn_id",
+ s3_bucket=S3_BUCKET,
+ s3_key=S3_KEY,
+ )
+ # [END howto_sftp_transfer_data_to_s3]
diff --git a/airflow/providers/amazon/aws/transfers/s3_to_sftp.py b/airflow/providers/amazon/aws/transfers/s3_to_sftp.py
index 486af07..9b8c0b3 100644
--- a/airflow/providers/amazon/aws/transfers/s3_to_sftp.py
+++ b/airflow/providers/amazon/aws/transfers/s3_to_sftp.py
@@ -28,6 +28,10 @@ class S3ToSFTPOperator(BaseOperator):
"""
This operator enables the transferring of files from S3 to a SFTP server.
+ .. seealso::
+ For more information on how to use this operator, take a look at the guide:
+ :ref:`howto/operator:S3ToSFTPOperator`
+
:param sftp_conn_id: The sftp connection id. The name or identifier for
establishing a connection to the SFTP server.
:type sftp_conn_id: str
diff --git a/airflow/providers/amazon/aws/transfers/sftp_to_s3.py b/airflow/providers/amazon/aws/transfers/sftp_to_s3.py
index a3e4d5b..dad2d85 100644
--- a/airflow/providers/amazon/aws/transfers/sftp_to_s3.py
+++ b/airflow/providers/amazon/aws/transfers/sftp_to_s3.py
@@ -28,6 +28,10 @@ class SFTPToS3Operator(BaseOperator):
This operator enables the transferring of files from a SFTP server to
Amazon S3.
+ .. seealso::
+ For more information on how to use this operator, take a look at the guide:
+ :ref:`howto/operator:SFTPToS3Operator`
+
:param sftp_conn_id: The sftp connection id. The name or identifier for
establishing a connection to the SFTP server.
:type sftp_conn_id: str
diff --git a/airflow/providers/amazon/provider.yaml b/airflow/providers/amazon/provider.yaml
index 2b895cc..ff07b2a 100644
--- a/airflow/providers/amazon/provider.yaml
+++ b/airflow/providers/amazon/provider.yaml
@@ -375,9 +375,11 @@ transfers:
python-module: airflow.providers.amazon.aws.transfers.s3_to_redshift
- source-integration-name: Amazon Simple Storage Service (S3)
target-integration-name: SSH File Transfer Protocol (SFTP)
+ how-to-guide: /docs/apache-airflow-providers-amazon/operators/transfer/s3_to_sftp.rst
python-module: airflow.providers.amazon.aws.transfers.s3_to_sftp
- source-integration-name: SSH File Transfer Protocol (SFTP)
target-integration-name: Amazon Simple Storage Service (S3)
+ how-to-guide: /docs/apache-airflow-providers-amazon/operators/transfer/sftp_to_s3.rst
python-module: airflow.providers.amazon.aws.transfers.sftp_to_s3
- source-integration-name: Amazon Simple Storage Service (S3)
target-integration-name: File Transfer Protocol (FTP)
diff --git a/docs/apache-airflow-providers-amazon/operators/transfer/s3_to_sftp.rst b/docs/apache-airflow-providers-amazon/operators/transfer/s3_to_sftp.rst
new file mode 100644
index 0000000..10b6632
--- /dev/null
+++ b/docs/apache-airflow-providers-amazon/operators/transfer/s3_to_sftp.rst
@@ -0,0 +1,42 @@
+ .. Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ .. http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+
+Amazon S3 to SFTP Transfer Operator
+===================================
+
+AWS Transfer for SFTP provides Secure File Transfer Protocol (SFTP) access to a customer's S3 resources.
+For more information about the service visits `Amazon Transfer for SFTP API documentation <https://docs.aws.amazon.com/whitepapers/latest/architecting-hipaa-security-and-compliance-on-aws/aws-transfer-for-sftp.html>`_
+
+
+.. _howto/operator:S3ToSFTPOperator:
+
+S3ToSFTPOperator
+^^^^^^^^^^^^^^^^
+
+This operator enables the transferring of files from S3 to a SFTP Server.
+
+To get more information about operator visit:
+:class:`~airflow.providers.amazon.aws.transfers.s3_to_sftp.S3ToSFTPOperator`
+
+Example usage:
+
+.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_s3_to_sftp.py
+ :language: python
+ :dedent: 4
+ :start-after: [START howto_s3_transfer_data_to_sftp]
+ :end-before: [END howto_s3_transfer_data_to_sftp]
diff --git a/docs/apache-airflow-providers-amazon/operators/transfer/sftp_to_s3.rst b/docs/apache-airflow-providers-amazon/operators/transfer/sftp_to_s3.rst
new file mode 100644
index 0000000..e0d1206
--- /dev/null
+++ b/docs/apache-airflow-providers-amazon/operators/transfer/sftp_to_s3.rst
@@ -0,0 +1,40 @@
+ .. Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ .. http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+
+Amazon SFTP to S3 Transfer Operator
+===================================
+
+AWS Transfer for SFTP provides Secure File Transfer Protocol (SFTP) access to a customer's S3 resources. For more information about the service visit `Amazon Transfer for SFTP API documentation <https://docs.aws.amazon.com/whitepapers/latest/architecting-hipaa-security-and-compliance-on-aws/aws-transfer-for-sftp.html>`_
+
+.. _howto/operator:SFTPToS3Operator:
+
+SFTPToS3Operator
+^^^^^^^^^^^^^^^^
+
+This operator enables the transferring of files from a SFTP server to Amazon S3.
+
+To get more information about operator visit:
+:class:`~airflow.providers.amazon.aws.transfers.sftp_to_s3.SFTPToS3Operator`
+
+Example usage:
+
+.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_sftp_to_s3.py
+ :language: python
+ :dedent: 4
+ :start-after: [START howto_sftp_transfer_data_to_s3]
+ :end-before: [END howto_sftp_transfer_data_to_s3]