You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2022/03/27 20:18:13 UTC
[airflow] branch main updated: Add doc and example dag for AWS CloudFormation Operators (#22533)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 02526b3 Add doc and example dag for AWS CloudFormation Operators (#22533)
02526b3 is described below
commit 02526b3f64d090e812ebaf3c37a23da2a3e3084e
Author: Niko <on...@amazon.com>
AuthorDate: Sun Mar 27 13:17:26 2022 -0700
Add doc and example dag for AWS CloudFormation Operators (#22533)
---
.../aws/example_dags/example_cloudformation.py | 83 ++++++++++++++++++
.../amazon/aws/operators/cloud_formation.py | 14 ++--
.../amazon/aws/sensors/cloud_formation.py | 10 +++
airflow/providers/amazon/provider.yaml | 2 +
.../operators/cloudformation.rst | 98 ++++++++++++++++++++++
5 files changed, 202 insertions(+), 5 deletions(-)
diff --git a/airflow/providers/amazon/aws/example_dags/example_cloudformation.py b/airflow/providers/amazon/aws/example_dags/example_cloudformation.py
new file mode 100644
index 0000000..c31d321
--- /dev/null
+++ b/airflow/providers/amazon/aws/example_dags/example_cloudformation.py
@@ -0,0 +1,83 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from datetime import datetime
+from json import dumps
+
+from airflow import DAG
+from airflow.models.baseoperator import chain
+from airflow.providers.amazon.aws.operators.cloud_formation import (
+ CloudFormationCreateStackOperator,
+ CloudFormationDeleteStackOperator,
+)
+from airflow.providers.amazon.aws.sensors.cloud_formation import (
+ CloudFormationCreateStackSensor,
+ CloudFormationDeleteStackSensor,
+)
+
+CLOUDFORMATION_STACK_NAME = 'example-stack-name'
+# The CloudFormation template must have at least one resource to be usable, this example uses SQS
+# as a free and serverless option.
+CLOUDFORMATION_TEMPLATE = {
+ 'Description': 'Stack from Airflow CloudFormation example DAG',
+ 'Resources': {
+ 'ExampleQueue': {
+ 'Type': 'AWS::SQS::Queue',
+ }
+ },
+}
+CLOUDFORMATION_CREATE_PARAMETERS = {
+ 'StackName': CLOUDFORMATION_STACK_NAME,
+ 'TemplateBody': dumps(CLOUDFORMATION_TEMPLATE),
+ 'TimeoutInMinutes': 2,
+ 'OnFailure': 'DELETE', # Don't leave stacks behind if creation fails.
+}
+
+with DAG(
+ dag_id='example_cloudformation',
+ schedule_interval=None,
+ start_date=datetime(2021, 1, 1),
+ tags=['example'],
+ catchup=False,
+) as dag:
+
+ # [START howto_operator_cloudformation_create_stack]
+ create_stack = CloudFormationCreateStackOperator(
+ task_id='create_stack',
+ stack_name=CLOUDFORMATION_STACK_NAME,
+ cloudformation_parameters=CLOUDFORMATION_CREATE_PARAMETERS,
+ )
+ # [END howto_operator_cloudformation_create_stack]
+
+ # [START howto_sensor_cloudformation_create_stack]
+ wait_for_stack_create = CloudFormationCreateStackSensor(
+ task_id='wait_for_stack_creation', stack_name=CLOUDFORMATION_STACK_NAME
+ )
+ # [END howto_sensor_cloudformation_create_stack]
+
+ # [START howto_operator_cloudformation_delete_stack]
+ delete_stack = CloudFormationDeleteStackOperator(
+ task_id='delete_stack', stack_name=CLOUDFORMATION_STACK_NAME
+ )
+ # [END howto_operator_cloudformation_delete_stack]
+
+ # [START howto_sensor_cloudformation_delete_stack]
+ wait_for_stack_delete = CloudFormationDeleteStackSensor(
+ task_id='wait_for_stack_deletion', trigger_rule='all_done', stack_name=CLOUDFORMATION_STACK_NAME
+ )
+ # [END howto_sensor_cloudformation_delete_stack]
+
+ chain(create_stack, wait_for_stack_create, delete_stack, wait_for_stack_delete)
diff --git a/airflow/providers/amazon/aws/operators/cloud_formation.py b/airflow/providers/amazon/aws/operators/cloud_formation.py
index 677ac81..2e7bb8a 100644
--- a/airflow/providers/amazon/aws/operators/cloud_formation.py
+++ b/airflow/providers/amazon/aws/operators/cloud_formation.py
@@ -29,11 +29,13 @@ class CloudFormationCreateStackOperator(BaseOperator):
"""
An operator that creates a CloudFormation stack.
+ .. seealso::
+ For more information on how to use this operator, take a look at the guide:
+ :ref:`howto/operator:CloudFormationCreateStackOperator`
+
+
:param stack_name: stack name (templated)
:param cloudformation_parameters: parameters to be passed to CloudFormation.
-
- .. seealso::
- https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/cloudformation.html#CloudFormation.Client.create_stack
:param aws_conn_id: aws connection to uses
"""
@@ -63,8 +65,10 @@ class CloudFormationDeleteStackOperator(BaseOperator):
:param stack_name: stack name (templated)
:param cloudformation_parameters: parameters to be passed to CloudFormation.
- .. seealso::
- https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/cloudformation.html#CloudFormation.Client.delete_stack
+ .. seealso::
+ For more information on how to use this operator, take a look at the guide:
+ :ref:`howto/operator:CloudFormationDeleteStackOperator`
+
:param aws_conn_id: aws connection to uses
"""
diff --git a/airflow/providers/amazon/aws/sensors/cloud_formation.py b/airflow/providers/amazon/aws/sensors/cloud_formation.py
index bf78b3b..2add211 100644
--- a/airflow/providers/amazon/aws/sensors/cloud_formation.py
+++ b/airflow/providers/amazon/aws/sensors/cloud_formation.py
@@ -35,6 +35,11 @@ class CloudFormationCreateStackSensor(BaseSensorOperator):
"""
Waits for a stack to be created successfully on AWS CloudFormation.
+ .. seealso::
+ For more information on how to use this operator, take a look at the guide:
+ :ref:`howto/operator:CloudFormationCreateStackSensor`
+
+
:param stack_name: The name of the stack to wait for (templated)
:param aws_conn_id: ID of the Airflow connection where credentials and extra configuration are
stored
@@ -68,6 +73,11 @@ class CloudFormationDeleteStackSensor(BaseSensorOperator):
"""
Waits for a stack to be deleted successfully on AWS CloudFormation.
+ .. seealso::
+ For more information on how to use this operator, take a look at the guide:
+ :ref:`howto/operator:CloudFormationDeleteStackSensor`
+
+
:param stack_name: The name of the stack to wait for (templated)
:param aws_conn_id: ID of the Airflow connection where credentials and extra configuration are
stored
diff --git a/airflow/providers/amazon/provider.yaml b/airflow/providers/amazon/provider.yaml
index 8e6ff32..aeaf630 100644
--- a/airflow/providers/amazon/provider.yaml
+++ b/airflow/providers/amazon/provider.yaml
@@ -51,6 +51,8 @@ integrations:
- integration-name: Amazon CloudFormation
external-doc-url: https://aws.amazon.com/cloudformation/
logo: /integration-logos/aws/AWS-CloudFormation_light-bg@4x.png
+ how-to-guide:
+ - /docs/apache-airflow-providers-amazon/operators/cloudformation.rst
tags: [aws]
- integration-name: Amazon CloudWatch Logs
external-doc-url: https://aws.amazon.com/cloudwatch/
diff --git a/docs/apache-airflow-providers-amazon/operators/cloudformation.rst b/docs/apache-airflow-providers-amazon/operators/cloudformation.rst
new file mode 100644
index 0000000..9ddb153
--- /dev/null
+++ b/docs/apache-airflow-providers-amazon/operators/cloudformation.rst
@@ -0,0 +1,98 @@
+ .. Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ .. http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+
+AWS CloudFormation Operators
+============================
+
+`AWS CloudFormation <https://aws.amazon.com/cloudformation/>`__ enables you to create and provision AWS
+infrastructure deployments predictably and repeatedly. It helps you leverage AWS products such as Amazon
+EC2, Amazon Elastic Block Store, Amazon SNS, Elastic Load Balancing, and Auto Scaling to build highly
+reliable, highly scalable, cost-effective applications in the cloud without worrying about creating and
+configuring the underlying AWS infrastructure. AWS CloudFormation enables you to use a template file to
+create and delete a collection of resources together as a single unit (a stack).
+
+Prerequisite Tasks
+------------------
+
+.. include:: _partials/prerequisite_tasks.rst
+
+.. _howto/operator:CloudFormationCreateStackOperator:
+
+AWS CloudFormation Create Stack Operator
+""""""""""""""""""""""""""""""""""""""""
+
+To create a new AWS CloudFormation stack use
+:class:`~airflow.providers.amazon.aws.operators.cloud_formation.CloudFormationCreateStackOperator`.
+
+.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_cloudformation.py
+ :language: python
+ :dedent: 4
+ :start-after: [START howto_operator_cloudformation_create_stack]
+ :end-before: [END howto_operator_cloudformation_create_stack]
+
+
+.. _howto/operator:CloudFormationCreateStackSensor:
+
+AWS CloudFormation Create Stack Sensor
+""""""""""""""""""""""""""""""""""""""
+
+To wait on the state of an AWS CloudFormation stack creation until it reaches a terminal state you can use
+:class:`~airflow.providers.amazon.aws.sensors.cloud_formation.CloudFormationCreateStackSensor`
+
+.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_cloudformation.py
+ :language: python
+ :dedent: 4
+ :start-after: [START howto_sensor_cloudformation_create_stack]
+ :end-before: [END howto_sensor_cloudformation_create_stack]
+
+.. _howto/operator:CloudFormationDeleteStackOperator:
+
+AWS CloudFormation Delete Stack Operator
+""""""""""""""""""""""""""""""""""""""""
+
+To delete an AWS CloudFormation stack you can use
+:class:`~airflow.providers.amazon.aws.operators.cloud_formation.CloudFormationDeleteStackOperator`.
+
+.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_cloudformation.py
+ :language: python
+ :dedent: 4
+ :start-after: [START howto_operator_cloudformation_delete_stack]
+ :end-before: [END howto_operator_cloudformation_delete_stack]
+
+
+.. _howto/operator:CloudFormationDeleteStackSensor:
+
+AWS CloudFormation Delete Stack Sensor
+""""""""""""""""""""""""""""""""""""""
+
+To wait on the state of an AWS CloudFormation stack deletion until it reaches a terminal state you can use
+use :class:`~airflow.providers.amazon.aws.sensors.cloud_formation.CloudFormationDeleteStackSensor`
+
+.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_cloudformation.py
+ :language: python
+ :dedent: 4
+ :start-after: [START howto_sensor_cloudformation_delete_stack]
+ :end-before: [END howto_sensor_cloudformation_delete_stack]
+
+
+Reference
+---------
+
+For further information, look at:
+
+* `Boto3 Library Documentation for CloudFormation <https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/cloudformation.html>`__