You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2023/07/03 10:40:05 UTC
[airflow] branch main updated: Adds Sensor section in the Azure providers docs (#32299)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new c03d7b7c33 Adds Sensor section in the Azure providers docs (#32299)
c03d7b7c33 is described below
commit c03d7b7c337df7eee4cc7722a1c8da56abd7027a
Author: Akash Sharma <35...@users.noreply.github.com>
AuthorDate: Mon Jul 3 16:09:57 2023 +0530
Adds Sensor section in the Azure providers docs (#32299)
* added transfers section:
* spell
* spell
* spell
* static-checks
* fixed static checks
* doc added for local_to_wasb
* added sensors section
* added sensors section
* fixed static check
---
.../microsoft/azure/example_dag/__init__.py | 16 ++++++
.../example_dag/example_cosmos_document_sensor.py | 53 +++++++++++++++++++
.../azure/example_dag/example_wasb_sensors.py | 61 ++++++++++++++++++++++
.../index.rst | 1 +
.../sensors/cosmos_document_sesnor.rst | 35 +++++++++++++
.../sensors/index.rst | 27 ++++++++++
.../sensors/wasb_sensors.rst | 48 +++++++++++++++++
docs/spelling_wordlist.txt | 1 +
8 files changed, 242 insertions(+)
diff --git a/airflow/providers/microsoft/azure/example_dag/__init__.py b/airflow/providers/microsoft/azure/example_dag/__init__.py
new file mode 100644
index 0000000000..13a83393a9
--- /dev/null
+++ b/airflow/providers/microsoft/azure/example_dag/__init__.py
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
diff --git a/airflow/providers/microsoft/azure/example_dag/example_cosmos_document_sensor.py b/airflow/providers/microsoft/azure/example_dag/example_cosmos_document_sensor.py
new file mode 100644
index 0000000000..198420147e
--- /dev/null
+++ b/airflow/providers/microsoft/azure/example_dag/example_cosmos_document_sensor.py
@@ -0,0 +1,53 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""
+Example Airflow DAG that senses document in Azure Cosmos DB.
+
+This DAG relies on the following OS environment variables
+
+* DATABASE_NAME - Target CosmosDB database_name.
+* COLLECTION_NAME - Target CosmosDB collection_name.
+* DOCUMENT_ID - The ID of the target document.
+"""
+from __future__ import annotations
+
+import os
+from datetime import datetime
+
+from airflow.models import DAG
+from airflow.providers.microsoft.azure.sensors.cosmos import AzureCosmosDocumentSensor
+
+DATABASE_NAME = os.environ.get("DATABASE_NAME", "example-database-name")
+COLLECTION_NAME = os.environ.get("COLLECTION_NAME", "example-collection-name")
+DOCUMENT_ID = os.environ.get("DOCUMENT_ID", "example-document-id")
+
+
+with DAG(
+ "example_cosmos_document_sensor",
+ start_date=datetime(2022, 8, 8),
+ catchup=False,
+ tags=["example"],
+) as dag:
+ # [START cosmos_document_sensor]
+ azure_wasb_sensor = AzureCosmosDocumentSensor(
+ database_name=DATABASE_NAME,
+ collection_name=COLLECTION_NAME,
+ document_id=DOCUMENT_ID,
+ task_id="cosmos_document_sensor",
+ )
+ # [END cosmos_document_sensor]
diff --git a/airflow/providers/microsoft/azure/example_dag/example_wasb_sensors.py b/airflow/providers/microsoft/azure/example_dag/example_wasb_sensors.py
new file mode 100644
index 0000000000..349e09c44a
--- /dev/null
+++ b/airflow/providers/microsoft/azure/example_dag/example_wasb_sensors.py
@@ -0,0 +1,61 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""
+Example Airflow DAG that senses blob(s) in Azure Blob Storage.
+
+This DAG relies on the following OS environment variables
+
+* CONTAINER_NAME - The container under which to look for the blob.
+* BLOB_NAME - The name of the blob to match.
+* PREFIX - The blob with the specified prefix to match.
+"""
+from __future__ import annotations
+
+import os
+from datetime import datetime
+
+from airflow.models import DAG
+from airflow.providers.microsoft.azure.sensors.wasb import WasbBlobSensor, WasbPrefixSensor
+
+CONTAINER_NAME = os.environ.get("CONTAINER_NAME", "example-container-name")
+BLOB_NAME = os.environ.get("BLOB_NAME", "example-blob-name")
+PREFIX = os.environ.get("PREFIX", "example-prefix")
+
+
+with DAG(
+ "example_wasb_sensors",
+ start_date=datetime(2022, 8, 8),
+ catchup=False,
+ tags=["example"],
+) as dag:
+ # [START wasb_blob_sensor]
+ azure_wasb_sensor = WasbBlobSensor(
+ container_name=CONTAINER_NAME,
+ blob_name=BLOB_NAME,
+ task_id="wasb_sense_blob",
+ )
+ # [END wasb_blob_sensor]
+
+ # [START wasb_prefix_sensor]
+ azure_wasb_prefix_sensor = WasbPrefixSensor(
+ container_name=CONTAINER_NAME,
+ blob_name=BLOB_NAME,
+ prefix=PREFIX,
+ task_id="wasb_sense_prefix",
+ )
+ # [END wasb_prefix_sensor]
diff --git a/docs/apache-airflow-providers-microsoft-azure/index.rst b/docs/apache-airflow-providers-microsoft-azure/index.rst
index dfc0bf56e1..a10c501731 100644
--- a/docs/apache-airflow-providers-microsoft-azure/index.rst
+++ b/docs/apache-airflow-providers-microsoft-azure/index.rst
@@ -39,6 +39,7 @@
Transfers <transfer/index>
Secrets backends <secrets-backends/azure-key-vault>
Logging for Tasks <logging/index>
+ Sensors <sensors/index>
.. toctree::
:hidden:
diff --git a/docs/apache-airflow-providers-microsoft-azure/sensors/cosmos_document_sesnor.rst b/docs/apache-airflow-providers-microsoft-azure/sensors/cosmos_document_sesnor.rst
new file mode 100644
index 0000000000..d4b5631204
--- /dev/null
+++ b/docs/apache-airflow-providers-microsoft-azure/sensors/cosmos_document_sesnor.rst
@@ -0,0 +1,35 @@
+ .. Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ .. http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+Azure Cosmos DB
+==================
+Cosmos Database (DB) is a globally distributed, low latency, multi-model database for managing data at large scales.
+It is a cloud-based NoSQL database offered as a PaaS (Platform as a Service) from Microsoft Azure.
+It is a highly available, high throughput, reliable database and is often called a serverless database.
+Cosmos database contains the Azure Document DB and is available everywhere.
+
+Azure Cosmos Document Sensor
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Checks for the existence of a document which matches the given query in CosmosDB.
+:class:`~airflow.providers.microsoft.azure.sensors.cosmos.AzureCosmosDocumentSensor`
+
+.. exampleinclude:: /../../airflow/providers/microsoft/azure/example_dag/example_cosmos_document_sensor.py
+ :language: python
+ :dedent: 4
+ :start-after: [START cosmos_document_sensor]
+ :end-before: [END cosmos_document_sensor]
diff --git a/docs/apache-airflow-providers-microsoft-azure/sensors/index.rst b/docs/apache-airflow-providers-microsoft-azure/sensors/index.rst
new file mode 100644
index 0000000000..439d234897
--- /dev/null
+++ b/docs/apache-airflow-providers-microsoft-azure/sensors/index.rst
@@ -0,0 +1,27 @@
+ .. Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ .. http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+
+
+Microsoft Azure Sensors
+=======================
+
+.. toctree::
+ :maxdepth: 1
+ :glob:
+
+ *
diff --git a/docs/apache-airflow-providers-microsoft-azure/sensors/wasb_sensors.rst b/docs/apache-airflow-providers-microsoft-azure/sensors/wasb_sensors.rst
new file mode 100644
index 0000000000..f901928d9b
--- /dev/null
+++ b/docs/apache-airflow-providers-microsoft-azure/sensors/wasb_sensors.rst
@@ -0,0 +1,48 @@
+ .. Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ .. http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+Azure Blob Storage
+==================
+The Blob service stores text and binary data as objects in the cloud.
+The Blob service offers the following three resources: the storage account, containers, and blobs.
+Within your storage account, containers provide a way to organize sets of blobs.
+For more information about the service visit `Azure Blob Storage API documentation <https://docs.microsoft.com/en-us/rest/api/storageservices/blob-service-rest-api>`_.
+This page shows how to check for blobs in a particular container.
+
+Wasb Blob Sensor
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Waits for a blob to arrive on Azure Blob Storage.
+:class:`~airflow.providers.microsoft.azure.sensors.wasb.WasbBlobSensor`
+
+.. exampleinclude:: /../../airflow/providers/microsoft/azure/example_dag/example_wasb_sensors.py
+ :language: python
+ :dedent: 4
+ :start-after: [START wasb_blob_sensor]
+ :end-before: [END wasb_blob_sensor]
+
+Wasb Prefix Sensor
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Waits for blobs matching a prefix to arrive on Azure Blob Storage.
+:class:`~airflow.providers.microsoft.azure.sensors.wasb.WasbPrefixSensor`
+
+.. exampleinclude:: /../../airflow/providers/microsoft/azure/example_dag/example_wasb_sensors.py
+ :language: python
+ :dedent: 4
+ :start-after: [START wasb_prefix_sensor]
+ :end-before: [END wasb_prefix_sensor]
diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt
index af0a3ff5dc..6389ba1814 100644
--- a/docs/spelling_wordlist.txt
+++ b/docs/spelling_wordlist.txt
@@ -1056,6 +1056,7 @@ ot
otel
overridable
oversubscription
+PaaS
Pagerduty
pagerduty
pageviews