You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2022/05/04 20:42:54 UTC
[airflow] branch main updated: CloudTasks assets & system tests migration (AIP-47) (#23282)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 3977e1798d CloudTasks assets & system tests migration (AIP-47) (#23282)
3977e1798d is described below
commit 3977e1798d8294ba628b5f330f43702c1a5c79fc
Author: Bartłomiej Hirsz <ba...@gmail.com>
AuthorDate: Wed May 4 22:42:45 2022 +0200
CloudTasks assets & system tests migration (AIP-47) (#23282)
---
airflow/providers/google/cloud/hooks/tasks.py | 6 +-
.../providers/google/cloud/links/cloud_tasks.py | 81 +++++++++++
airflow/providers/google/cloud/operators/tasks.py | 69 ++++++++-
airflow/providers/google/provider.yaml | 2 +
.../operators/cloud/tasks.rst | 26 ++--
.../providers/google/cloud/operators/test_tasks.py | 26 ++--
.../google/cloud/operators/test_tasks_system.py | 34 -----
.../system/providers/google/tasks/example_queue.py | 116 ++++++---------
.../providers/google/tasks}/example_tasks.py | 157 ++++++++-------------
9 files changed, 282 insertions(+), 235 deletions(-)
diff --git a/airflow/providers/google/cloud/hooks/tasks.py b/airflow/providers/google/cloud/hooks/tasks.py
index 09ed9b0112..2265e36a79 100644
--- a/airflow/providers/google/cloud/hooks/tasks.py
+++ b/airflow/providers/google/cloud/hooks/tasks.py
@@ -298,7 +298,7 @@ class CloudTasksHook(GoogleBaseHook):
retry: Union[Retry, _MethodDefault] = DEFAULT,
timeout: Optional[float] = None,
metadata: Sequence[Tuple[str, str]] = (),
- ) -> List[Queue]:
+ ) -> Queue:
"""
Purges a queue by deleting all of its tasks from Cloud Tasks.
@@ -333,7 +333,7 @@ class CloudTasksHook(GoogleBaseHook):
retry: Union[Retry, _MethodDefault] = DEFAULT,
timeout: Optional[float] = None,
metadata: Sequence[Tuple[str, str]] = (),
- ) -> List[Queue]:
+ ) -> Queue:
"""
Pauses a queue in Cloud Tasks.
@@ -368,7 +368,7 @@ class CloudTasksHook(GoogleBaseHook):
retry: Union[Retry, _MethodDefault] = DEFAULT,
timeout: Optional[float] = None,
metadata: Sequence[Tuple[str, str]] = (),
- ) -> List[Queue]:
+ ) -> Queue:
"""
Resumes a queue in Cloud Tasks.
diff --git a/airflow/providers/google/cloud/links/cloud_tasks.py b/airflow/providers/google/cloud/links/cloud_tasks.py
new file mode 100644
index 0000000000..16a28ee4b6
--- /dev/null
+++ b/airflow/providers/google/cloud/links/cloud_tasks.py
@@ -0,0 +1,81 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""This module contains Google Cloud Tasks links."""
+from typing import TYPE_CHECKING, Optional
+
+from airflow.models import BaseOperator
+from airflow.providers.google.cloud.links.base import BaseGoogleLink
+
+if TYPE_CHECKING:
+ from airflow.utils.context import Context
+
+CLOUD_TASKS_BASE_LINK = "https://pantheon.corp.google.com/cloudtasks"
+CLOUD_TASKS_QUEUE_LINK = CLOUD_TASKS_BASE_LINK + "/queue/{location}/{queue_id}/tasks?project={project_id}"
+CLOUD_TASKS_LINK = CLOUD_TASKS_BASE_LINK + "?project={project_id}"
+
+
+class CloudTasksQueueLink(BaseGoogleLink):
+ """Helper class for constructing Cloud Task Queue Link"""
+
+ name = "Cloud Tasks Queue"
+ key = "cloud_task_queue"
+ format_str = CLOUD_TASKS_QUEUE_LINK
+
+ @staticmethod
+ def extract_parts(queue_name: Optional[str]):
+ """
+ Extract project_id, location and queue id from queue name:
+ projects/PROJECT_ID/locations/LOCATION_ID/queues/QUEUE_ID
+ """
+ if not queue_name:
+ return "", "", ""
+ parts = queue_name.split("/")
+ return parts[1], parts[3], parts[5]
+
+ @staticmethod
+ def persist(
+ operator_instance: BaseOperator,
+ context: "Context",
+ queue_name: Optional[str],
+ ):
+ project_id, location, queue_id = CloudTasksQueueLink.extract_parts(queue_name)
+ operator_instance.xcom_push(
+ context,
+ key=CloudTasksQueueLink.key,
+ value={"project_id": project_id, "location": location, "queue_id": queue_id},
+ )
+
+
+class CloudTasksLink(BaseGoogleLink):
+ """Helper class for constructing Cloud Task Link"""
+
+ name = "Cloud Tasks"
+ key = "cloud_task"
+ format_str = CLOUD_TASKS_LINK
+
+ @staticmethod
+ def persist(
+ operator_instance: BaseOperator,
+ context: "Context",
+ project_id: Optional[str],
+ ):
+ operator_instance.xcom_push(
+ context,
+ key=CloudTasksLink.key,
+ value={"project_id": project_id},
+ )
diff --git a/airflow/providers/google/cloud/operators/tasks.py b/airflow/providers/google/cloud/operators/tasks.py
index 501b337521..b0c60309bb 100644
--- a/airflow/providers/google/cloud/operators/tasks.py
+++ b/airflow/providers/google/cloud/operators/tasks.py
@@ -31,6 +31,7 @@ from google.protobuf.field_mask_pb2 import FieldMask
from airflow.models import BaseOperator
from airflow.providers.google.cloud.hooks.tasks import CloudTasksHook
+from airflow.providers.google.cloud.links.cloud_tasks import CloudTasksLink, CloudTasksQueueLink
if TYPE_CHECKING:
from airflow.utils.context import Context
@@ -82,6 +83,7 @@ class CloudTasksQueueCreateOperator(BaseOperator):
"gcp_conn_id",
"impersonation_chain",
)
+ operator_extra_links = (CloudTasksQueueLink(),)
def __init__(
self,
@@ -134,7 +136,11 @@ class CloudTasksQueueCreateOperator(BaseOperator):
timeout=self.timeout,
metadata=self.metadata,
)
-
+ CloudTasksQueueLink.persist(
+ operator_instance=self,
+ context=context,
+ queue_name=queue.name,
+ )
return Queue.to_dict(queue)
@@ -186,6 +192,7 @@ class CloudTasksQueueUpdateOperator(BaseOperator):
"gcp_conn_id",
"impersonation_chain",
)
+ operator_extra_links = (CloudTasksQueueLink(),)
def __init__(
self,
@@ -229,6 +236,11 @@ class CloudTasksQueueUpdateOperator(BaseOperator):
timeout=self.timeout,
metadata=self.metadata,
)
+ CloudTasksQueueLink.persist(
+ operator_instance=self,
+ context=context,
+ queue_name=queue.name,
+ )
return Queue.to_dict(queue)
@@ -270,6 +282,7 @@ class CloudTasksQueueGetOperator(BaseOperator):
"gcp_conn_id",
"impersonation_chain",
)
+ operator_extra_links = (CloudTasksQueueLink(),)
def __init__(
self,
@@ -307,6 +320,11 @@ class CloudTasksQueueGetOperator(BaseOperator):
timeout=self.timeout,
metadata=self.metadata,
)
+ CloudTasksQueueLink.persist(
+ operator_instance=self,
+ context=context,
+ queue_name=queue.name,
+ )
return Queue.to_dict(queue)
@@ -349,6 +367,7 @@ class CloudTasksQueuesListOperator(BaseOperator):
"gcp_conn_id",
"impersonation_chain",
)
+ operator_extra_links = (CloudTasksLink(),)
def __init__(
self,
@@ -389,6 +408,11 @@ class CloudTasksQueuesListOperator(BaseOperator):
timeout=self.timeout,
metadata=self.metadata,
)
+ CloudTasksLink.persist(
+ operator_instance=self,
+ context=context,
+ project_id=self.project_id or hook.project_id,
+ )
return [Queue.to_dict(q) for q in queues]
@@ -505,6 +529,7 @@ class CloudTasksQueuePurgeOperator(BaseOperator):
"gcp_conn_id",
"impersonation_chain",
)
+ operator_extra_links = (CloudTasksQueueLink(),)
def __init__(
self,
@@ -542,6 +567,11 @@ class CloudTasksQueuePurgeOperator(BaseOperator):
timeout=self.timeout,
metadata=self.metadata,
)
+ CloudTasksQueueLink.persist(
+ operator_instance=self,
+ context=context,
+ queue_name=queue.name,
+ )
return Queue.to_dict(queue)
@@ -583,6 +613,7 @@ class CloudTasksQueuePauseOperator(BaseOperator):
"gcp_conn_id",
"impersonation_chain",
)
+ operator_extra_links = (CloudTasksQueueLink(),)
def __init__(
self,
@@ -620,6 +651,11 @@ class CloudTasksQueuePauseOperator(BaseOperator):
timeout=self.timeout,
metadata=self.metadata,
)
+ CloudTasksQueueLink.persist(
+ operator_instance=self,
+ context=context,
+ queue_name=queue.name,
+ )
return Queue.to_dict(queue)
@@ -661,6 +697,7 @@ class CloudTasksQueueResumeOperator(BaseOperator):
"gcp_conn_id",
"impersonation_chain",
)
+ operator_extra_links = (CloudTasksQueueLink(),)
def __init__(
self,
@@ -698,6 +735,11 @@ class CloudTasksQueueResumeOperator(BaseOperator):
timeout=self.timeout,
metadata=self.metadata,
)
+ CloudTasksQueueLink.persist(
+ operator_instance=self,
+ context=context,
+ queue_name=queue.name,
+ )
return Queue.to_dict(queue)
@@ -747,6 +789,7 @@ class CloudTasksTaskCreateOperator(BaseOperator):
"gcp_conn_id",
"impersonation_chain",
)
+ operator_extra_links = (CloudTasksQueueLink(),)
def __init__(
self,
@@ -793,6 +836,11 @@ class CloudTasksTaskCreateOperator(BaseOperator):
timeout=self.timeout,
metadata=self.metadata,
)
+ CloudTasksQueueLink.persist(
+ operator_instance=self,
+ context=context,
+ queue_name=task.name,
+ )
return Task.to_dict(task)
@@ -838,6 +886,7 @@ class CloudTasksTaskGetOperator(BaseOperator):
"gcp_conn_id",
"impersonation_chain",
)
+ operator_extra_links = (CloudTasksQueueLink(),)
def __init__(
self,
@@ -881,6 +930,11 @@ class CloudTasksTaskGetOperator(BaseOperator):
timeout=self.timeout,
metadata=self.metadata,
)
+ CloudTasksQueueLink.persist(
+ operator_instance=self,
+ context=context,
+ queue_name=task.name,
+ )
return Task.to_dict(task)
@@ -926,6 +980,7 @@ class CloudTasksTasksListOperator(BaseOperator):
"gcp_conn_id",
"impersonation_chain",
)
+ operator_extra_links = (CloudTasksQueueLink(),)
def __init__(
self,
@@ -969,6 +1024,12 @@ class CloudTasksTasksListOperator(BaseOperator):
timeout=self.timeout,
metadata=self.metadata,
)
+ CloudTasksQueueLink.persist(
+ operator_instance=self,
+ context=context,
+ queue_name=f"projects/{self.project_id or hook.project_id}/"
+ f"locations/{self.location}/queues/{self.queue_name}",
+ )
return [Task.to_dict(t) for t in tasks]
@@ -1094,6 +1155,7 @@ class CloudTasksTaskRunOperator(BaseOperator):
"gcp_conn_id",
"impersonation_chain",
)
+ operator_extra_links = (CloudTasksQueueLink(),)
def __init__(
self,
@@ -1137,4 +1199,9 @@ class CloudTasksTaskRunOperator(BaseOperator):
timeout=self.timeout,
metadata=self.metadata,
)
+ CloudTasksQueueLink.persist(
+ operator_instance=self,
+ context=context,
+ queue_name=task.name,
+ )
return Task.to_dict(task)
diff --git a/airflow/providers/google/provider.yaml b/airflow/providers/google/provider.yaml
index f806d64748..8cc1128b93 100644
--- a/airflow/providers/google/provider.yaml
+++ b/airflow/providers/google/provider.yaml
@@ -887,6 +887,8 @@ extra-links:
- airflow.providers.google.cloud.links.bigquery.BigQueryDatasetLink
- airflow.providers.google.cloud.links.bigquery.BigQueryTableLink
- airflow.providers.google.cloud.links.bigquery_dts.BigQueryDataTransferConfigLink
+ - airflow.providers.google.cloud.links.cloud_tasks.CloudTasksQueueLink
+ - airflow.providers.google.cloud.links.cloud_tasks.CloudTasksLink
- airflow.providers.google.cloud.links.dataproc.DataprocLink
- airflow.providers.google.cloud.links.dataproc.DataprocListLink
- airflow.providers.google.cloud.operators.dataproc_metastore.DataprocMetastoreDetailedLink
diff --git a/docs/apache-airflow-providers-google/operators/cloud/tasks.rst b/docs/apache-airflow-providers-google/operators/cloud/tasks.rst
index 037b93ca0c..875ae81920 100644
--- a/docs/apache-airflow-providers-google/operators/cloud/tasks.rst
+++ b/docs/apache-airflow-providers-google/operators/cloud/tasks.rst
@@ -41,7 +41,7 @@ Create queue
To create new Queue use
:class:`~airflow.providers.google.cloud.operators.tasks.CloudTasksQueueCreateOperator`
-.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_tasks.py
+.. exampleinclude:: /../../tests/system/providers/google/tasks/example_queue.py
:language: python
:dedent: 4
:start-after: [START create_queue]
@@ -55,7 +55,7 @@ Delete queue
To delete Queue use
:class:`~airflow.providers.google.cloud.operators.tasks.CloudTasksQueueDeleteOperator`
-.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_tasks.py
+.. exampleinclude:: /../../tests/system/providers/google/tasks/example_queue.py
:language: python
:dedent: 4
:start-after: [START delete_queue]
@@ -70,7 +70,7 @@ Resume queue
To resume Queue use
:class:`~airflow.providers.google.cloud.operators.tasks.CloudTasksQueueResumeOperator`
-.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_tasks.py
+.. exampleinclude:: /../../tests/system/providers/google/tasks/example_queue.py
:language: python
:dedent: 4
:start-after: [START resume_queue]
@@ -84,7 +84,7 @@ Pause queue
To pause Queue use
:class:`~airflow.providers.google.cloud.operators.tasks.CloudTasksQueuePauseOperator`
-.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_tasks.py
+.. exampleinclude:: /../../tests/system/providers/google/tasks/example_queue.py
:language: python
:dedent: 4
:start-after: [START pause_queue]
@@ -98,7 +98,7 @@ Purge queue
To purge Queue use
:class:`~airflow.providers.google.cloud.operators.tasks.CloudTasksQueuePurgeOperator`
-.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_tasks.py
+.. exampleinclude:: /../../tests/system/providers/google/tasks/example_queue.py
:language: python
:dedent: 4
:start-after: [START purge_queue]
@@ -112,7 +112,7 @@ Get queue
To get Queue use
:class:`~airflow.providers.google.cloud.operators.tasks.CloudTasksQueueGetOperator`
-.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_tasks.py
+.. exampleinclude:: /../../tests/system/providers/google/tasks/example_queue.py
:language: python
:dedent: 4
:start-after: [START get_queue]
@@ -126,7 +126,7 @@ Update queue
To update Queue use
:class:`~airflow.providers.google.cloud.operators.tasks.CloudTasksQueueUpdateOperator`
-.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_tasks.py
+.. exampleinclude:: /../../tests/system/providers/google/tasks/example_queue.py
:language: python
:dedent: 4
:start-after: [START update_queue]
@@ -140,7 +140,7 @@ List queues
To list all Queues use
:class:`~airflow.providers.google.cloud.operators.tasks.CloudTasksQueuesListOperator`
-.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_tasks.py
+.. exampleinclude:: /../../tests/system/providers/google/tasks/example_queue.py
:language: python
:dedent: 4
:start-after: [START list_queue]
@@ -158,7 +158,7 @@ Create task
To create new Task in a particular queue use
:class:`~airflow.providers.google.cloud.operators.tasks.CloudTasksTaskCreateOperator`
-.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_tasks.py
+.. exampleinclude:: /../../tests/system/providers/google/tasks/example_tasks.py
:language: python
:dedent: 4
:start-after: [START create_task]
@@ -172,7 +172,7 @@ Get task
To get the Tasks in a particular queue use
:class:`~airflow.providers.google.cloud.operators.tasks.CloudTasksTaskGetOperator`
-.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_tasks.py
+.. exampleinclude:: /../../tests/system/providers/google/tasks/example_tasks.py
:language: python
:dedent: 4
:start-after: [START tasks_get]
@@ -186,7 +186,7 @@ Run task
To run the Task in a particular queue use
:class:`~airflow.providers.google.cloud.operators.tasks.CloudTasksTaskRunOperator`
-.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_tasks.py
+.. exampleinclude:: /../../tests/system/providers/google/tasks/example_tasks.py
:language: python
:dedent: 4
:start-after: [START run_task]
@@ -200,7 +200,7 @@ List tasks
To list all Tasks in a particular queue use
:class:`~airflow.providers.google.cloud.operators.tasks.CloudTasksTasksListOperator`
-.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_tasks.py
+.. exampleinclude:: /../../tests/system/providers/google/tasks/example_tasks.py
:language: python
:dedent: 4
:start-after: [START list_tasks]
@@ -214,7 +214,7 @@ Delete task
To delete the Task from particular queue use
:class:`~airflow.providers.google.cloud.operators.tasks.CloudTasksTaskDeleteOperator`
-.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_tasks.py
+.. exampleinclude:: /../../tests/system/providers/google/tasks/example_tasks.py
:language: python
:dedent: 4
:start-after: [START create_task]
diff --git a/tests/providers/google/cloud/operators/test_tasks.py b/tests/providers/google/cloud/operators/test_tasks.py
index 6f252b0c00..b13be1ca8d 100644
--- a/tests/providers/google/cloud/operators/test_tasks.py
+++ b/tests/providers/google/cloud/operators/test_tasks.py
@@ -56,7 +56,7 @@ class TestCloudTasksQueueCreate(unittest.TestCase):
mock_hook.return_value.create_queue.return_value = TEST_QUEUE
operator = CloudTasksQueueCreateOperator(location=LOCATION, task_queue=TEST_QUEUE, task_id="id")
- result = operator.execute(context=None)
+ result = operator.execute(context=mock.MagicMock())
assert {'name': FULL_QUEUE_PATH, 'state': 0} == result
mock_hook.assert_called_once_with(
@@ -80,7 +80,7 @@ class TestCloudTasksQueueUpdate(unittest.TestCase):
mock_hook.return_value.update_queue.return_value = TEST_QUEUE
operator = CloudTasksQueueUpdateOperator(task_queue=Queue(name=FULL_QUEUE_PATH), task_id="id")
- result = operator.execute(context=None)
+ result = operator.execute(context=mock.MagicMock())
assert {'name': FULL_QUEUE_PATH, 'state': 0} == result
mock_hook.assert_called_once_with(
@@ -105,7 +105,7 @@ class TestCloudTasksQueueGet(unittest.TestCase):
mock_hook.return_value.get_queue.return_value = TEST_QUEUE
operator = CloudTasksQueueGetOperator(location=LOCATION, queue_name=QUEUE_ID, task_id="id")
- result = operator.execute(context=None)
+ result = operator.execute(context=mock.MagicMock())
assert {'name': FULL_QUEUE_PATH, 'state': 0} == result
mock_hook.assert_called_once_with(
@@ -128,7 +128,7 @@ class TestCloudTasksQueuesList(unittest.TestCase):
mock_hook.return_value.list_queues.return_value = [TEST_QUEUE]
operator = CloudTasksQueuesListOperator(location=LOCATION, task_id="id")
- result = operator.execute(context=None)
+ result = operator.execute(context=mock.MagicMock())
assert [{'name': FULL_QUEUE_PATH, 'state': 0}] == result
mock_hook.assert_called_once_with(
@@ -152,7 +152,7 @@ class TestCloudTasksQueueDelete(unittest.TestCase):
mock_hook.return_value.delete_queue.return_value = None
operator = CloudTasksQueueDeleteOperator(location=LOCATION, queue_name=QUEUE_ID, task_id="id")
- operator.execute(context=None)
+ operator.execute(context=mock.MagicMock())
mock_hook.assert_called_once_with(
gcp_conn_id=GCP_CONN_ID,
@@ -174,7 +174,7 @@ class TestCloudTasksQueuePurge(unittest.TestCase):
mock_hook.return_value.purge_queue.return_value = TEST_QUEUE
operator = CloudTasksQueuePurgeOperator(location=LOCATION, queue_name=QUEUE_ID, task_id="id")
- result = operator.execute(context=None)
+ result = operator.execute(context=mock.MagicMock())
assert {'name': FULL_QUEUE_PATH, 'state': 0} == result
mock_hook.assert_called_once_with(
@@ -197,7 +197,7 @@ class TestCloudTasksQueuePause(unittest.TestCase):
mock_hook.return_value.pause_queue.return_value = TEST_QUEUE
operator = CloudTasksQueuePauseOperator(location=LOCATION, queue_name=QUEUE_ID, task_id="id")
- result = operator.execute(context=None)
+ result = operator.execute(context=mock.MagicMock())
assert {'name': FULL_QUEUE_PATH, 'state': 0} == result
mock_hook.assert_called_once_with(
@@ -220,7 +220,7 @@ class TestCloudTasksQueueResume(unittest.TestCase):
mock_hook.return_value.resume_queue.return_value = TEST_QUEUE
operator = CloudTasksQueueResumeOperator(location=LOCATION, queue_name=QUEUE_ID, task_id="id")
- result = operator.execute(context=None)
+ result = operator.execute(context=mock.MagicMock())
assert {'name': FULL_QUEUE_PATH, 'state': 0} == result
mock_hook.assert_called_once_with(
@@ -245,7 +245,7 @@ class TestCloudTasksTaskCreate(unittest.TestCase):
location=LOCATION, queue_name=QUEUE_ID, task=Task(), task_id="id"
)
- result = operator.execute(context=None)
+ result = operator.execute(context=mock.MagicMock())
assert {
'app_engine_http_request': {'body': '', 'headers': {}, 'http_method': 0, 'relative_uri': ''},
@@ -279,7 +279,7 @@ class TestCloudTasksTaskGet(unittest.TestCase):
location=LOCATION, queue_name=QUEUE_ID, task_name=TASK_NAME, task_id="id"
)
- result = operator.execute(context=None)
+ result = operator.execute(context=mock.MagicMock())
assert {
'app_engine_http_request': {'body': '', 'headers': {}, 'http_method': 0, 'relative_uri': ''},
@@ -310,7 +310,7 @@ class TestCloudTasksTasksList(unittest.TestCase):
mock_hook.return_value.list_tasks.return_value = [TEST_TASK]
operator = CloudTasksTasksListOperator(location=LOCATION, queue_name=QUEUE_ID, task_id="id")
- result = operator.execute(context=None)
+ result = operator.execute(context=mock.MagicMock())
assert [
{
@@ -350,7 +350,7 @@ class TestCloudTasksTaskDelete(unittest.TestCase):
location=LOCATION, queue_name=QUEUE_ID, task_name=TASK_NAME, task_id="id"
)
- operator.execute(context=None)
+ operator.execute(context=mock.MagicMock())
mock_hook.assert_called_once_with(
gcp_conn_id=GCP_CONN_ID,
@@ -375,7 +375,7 @@ class TestCloudTasksTaskRun(unittest.TestCase):
location=LOCATION, queue_name=QUEUE_ID, task_name=TASK_NAME, task_id="id"
)
- result = operator.execute(context=None)
+ result = operator.execute(context=mock.MagicMock())
assert {
'app_engine_http_request': {'body': '', 'headers': {}, 'http_method': 0, 'relative_uri': ''},
diff --git a/tests/providers/google/cloud/operators/test_tasks_system.py b/tests/providers/google/cloud/operators/test_tasks_system.py
deleted file mode 100644
index 8d92ff7b6a..0000000000
--- a/tests/providers/google/cloud/operators/test_tasks_system.py
+++ /dev/null
@@ -1,34 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-import pytest
-
-from tests.providers.google.cloud.utils.gcp_authenticator import GCP_TASKS_KEY
-from tests.test_utils.gcp_system_helpers import CLOUD_DAG_FOLDER, GoogleSystemTest, provide_gcp_context
-
-
-@pytest.mark.credential_file(GCP_TASKS_KEY)
-class GcpTasksExampleDagsSystemTest(GoogleSystemTest):
- def setUp(self):
- super().setUp()
-
- @provide_gcp_context(GCP_TASKS_KEY)
- def test_run_example_dag_function(self):
- self.run_dag('example_gcp_tasks', CLOUD_DAG_FOLDER)
-
- def tearDown(self):
- super().tearDown()
diff --git a/airflow/providers/google/cloud/example_dags/example_tasks.py b/tests/system/providers/google/tasks/example_queue.py
similarity index 59%
copy from airflow/providers/google/cloud/example_dags/example_tasks.py
copy to tests/system/providers/google/tasks/example_queue.py
index 3d7ee739a4..61a83df4c6 100644
--- a/airflow/providers/google/cloud/example_dags/example_tasks.py
+++ b/tests/system/providers/google/tasks/example_queue.py
@@ -18,18 +18,17 @@
"""
Example Airflow DAG that creates, gets, lists, updates, purges, pauses, resumes
-and deletes Queues and creates, gets, lists, runs and deletes Tasks in the Google
-Cloud Tasks service in the Google Cloud.
+and deletes Queues in the Google Cloud Tasks service in the Google Cloud.
"""
import os
-from datetime import datetime, timedelta
+from datetime import datetime
from google.api_core.retry import Retry
from google.cloud.tasks_v2.types import Queue
-from google.protobuf import timestamp_pb2
from google.protobuf.field_mask_pb2 import FieldMask
from airflow import models
+from airflow.decorators import task
from airflow.models.baseoperator import chain
from airflow.operators.bash import BashOperator
from airflow.providers.google.cloud.operators.tasks import (
@@ -41,44 +40,43 @@ from airflow.providers.google.cloud.operators.tasks import (
CloudTasksQueueResumeOperator,
CloudTasksQueuesListOperator,
CloudTasksQueueUpdateOperator,
- CloudTasksTaskCreateOperator,
- CloudTasksTaskDeleteOperator,
- CloudTasksTaskGetOperator,
- CloudTasksTaskRunOperator,
- CloudTasksTasksListOperator,
)
+from airflow.utils.trigger_rule import TriggerRule
-timestamp = timestamp_pb2.Timestamp()
-timestamp.FromDatetime(datetime.now() + timedelta(hours=12))
+ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
+DAG_ID = "cloud_tasks_queue"
-LOCATION = "europe-west1"
-QUEUE_ID = os.environ.get('GCP_TASKS_QUEUE_ID', "cloud-tasks-queue")
-TASK_NAME = "task-to-run"
+LOCATION = "europe-central2"
+QUEUE_ID = f"queue-{ENV_ID}-{DAG_ID.replace('_', '-')}"
-TASK = {
- "app_engine_http_request": { # Specify the type of request.
- "http_method": "POST",
- "relative_uri": "/example_task_handler",
- "body": b"Hello",
- },
- "schedule_time": timestamp,
-}
-
with models.DAG(
- "example_gcp_tasks",
- schedule_interval='@once', # Override to match your needs
+ dag_id=DAG_ID,
+ schedule_interval='@once',
start_date=datetime(2021, 1, 1),
catchup=False,
- tags=['example'],
+ tags=['example', "tasks"],
) as dag:
- # Queue operations
+ @task(task_id="random_string")
+ def generate_random_string():
+ """
+ Generate random string for queue and task names.
+ Queue name cannot be repeated in preceding 7 days and
+ task name in the last 1 hour.
+ """
+ import random
+ import string
+
+ return "".join(random.choices(string.ascii_uppercase + string.digits, k=8))
+
+ random_string = generate_random_string()
+
# [START create_queue]
create_queue = CloudTasksQueueCreateOperator(
location=LOCATION,
task_queue=Queue(stackdriver_logging_config=dict(sampling_ratio=0.5)),
- queue_name=QUEUE_ID,
+ queue_name=QUEUE_ID + "{{ task_instance.xcom_pull(task_ids='random_string') }}",
retry=Retry(maximum=10.0),
timeout=5,
task_id="create_queue",
@@ -88,15 +86,16 @@ with models.DAG(
# [START delete_queue]
delete_queue = CloudTasksQueueDeleteOperator(
location=LOCATION,
- queue_name=QUEUE_ID,
+ queue_name=QUEUE_ID + "{{ task_instance.xcom_pull(task_ids='random_string') }}",
task_id="delete_queue",
)
# [END delete_queue]
+ delete_queue.trigger_rule = TriggerRule.ALL_DONE
# [START resume_queue]
resume_queue = CloudTasksQueueResumeOperator(
location=LOCATION,
- queue_name=QUEUE_ID,
+ queue_name=QUEUE_ID + "{{ task_instance.xcom_pull(task_ids='random_string') }}",
task_id="resume_queue",
)
# [END resume_queue]
@@ -104,7 +103,7 @@ with models.DAG(
# [START pause_queue]
pause_queue = CloudTasksQueuePauseOperator(
location=LOCATION,
- queue_name=QUEUE_ID,
+ queue_name=QUEUE_ID + "{{ task_instance.xcom_pull(task_ids='random_string') }}",
task_id="pause_queue",
)
# [END pause_queue]
@@ -112,7 +111,7 @@ with models.DAG(
# [START purge_queue]
purge_queue = CloudTasksQueuePurgeOperator(
location=LOCATION,
- queue_name=QUEUE_ID,
+ queue_name=QUEUE_ID + "{{ task_instance.xcom_pull(task_ids='random_string') }}",
task_id="purge_queue",
)
# [END purge_queue]
@@ -120,7 +119,7 @@ with models.DAG(
# [START get_queue]
get_queue = CloudTasksQueueGetOperator(
location=LOCATION,
- queue_name=QUEUE_ID,
+ queue_name=QUEUE_ID + "{{ task_instance.xcom_pull(task_ids='random_string') }}",
task_id="get_queue",
)
@@ -130,13 +129,11 @@ with models.DAG(
)
# [END get_queue]
- get_queue >> get_queue_result
-
# [START update_queue]
update_queue = CloudTasksQueueUpdateOperator(
task_queue=Queue(stackdriver_logging_config=dict(sampling_ratio=1)),
location=LOCATION,
- queue_name=QUEUE_ID,
+ queue_name=QUEUE_ID + "{{ task_instance.xcom_pull(task_ids='random_string') }}",
update_mask=FieldMask(paths=["stackdriver_logging_config.sampling_ratio"]),
task_id="update_queue",
)
@@ -147,55 +144,26 @@ with models.DAG(
# [END list_queue]
chain(
+ random_string,
create_queue,
update_queue,
pause_queue,
resume_queue,
purge_queue,
get_queue,
+ get_queue_result,
list_queue,
delete_queue,
)
- # Tasks operations
- # [START create_task]
- create_task = CloudTasksTaskCreateOperator(
- location=LOCATION,
- queue_name=QUEUE_ID,
- task=TASK,
- task_name=TASK_NAME,
- retry=Retry(maximum=10.0),
- timeout=5,
- task_id="create_task_to_run",
- )
- # [END create_task]
+ from tests.system.utils.watcher import watcher
- # [START tasks_get]
- tasks_get = CloudTasksTaskGetOperator(
- location=LOCATION,
- queue_name=QUEUE_ID,
- task_name=TASK_NAME,
- task_id="tasks_get",
- )
- # [END tasks_get]
+ # This test needs watcher in order to properly mark success/failure
+ # when "tearDown" task with trigger rule is part of the DAG
+ list(dag.tasks) >> watcher()
- # [START run_task]
- run_task = CloudTasksTaskRunOperator(
- location=LOCATION,
- queue_name=QUEUE_ID,
- task_name=TASK_NAME,
- task_id="run_task",
- )
- # [END run_task]
-
- # [START list_tasks]
- list_tasks = CloudTasksTasksListOperator(location=LOCATION, queue_name=QUEUE_ID, task_id="list_tasks")
- # [END list_tasks]
- # [START delete_task]
- delete_task = CloudTasksTaskDeleteOperator(
- location=LOCATION, queue_name=QUEUE_ID, task_name=TASK_NAME, task_id="delete_task"
- )
- # [END delete_task]
+from tests.system.utils import get_test_run # noqa: E402
- chain(purge_queue, create_task, tasks_get, list_tasks, run_task, delete_task, delete_queue)
+# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest)
+test_run = get_test_run(dag)
diff --git a/airflow/providers/google/cloud/example_dags/example_tasks.py b/tests/system/providers/google/tasks/example_tasks.py
similarity index 50%
rename from airflow/providers/google/cloud/example_dags/example_tasks.py
rename to tests/system/providers/google/tasks/example_tasks.py
index 3d7ee739a4..8ef2ac454b 100644
--- a/airflow/providers/google/cloud/example_dags/example_tasks.py
+++ b/tests/system/providers/google/tasks/example_tasks.py
@@ -17,9 +17,8 @@
# under the License.
"""
-Example Airflow DAG that creates, gets, lists, updates, purges, pauses, resumes
-and deletes Queues and creates, gets, lists, runs and deletes Tasks in the Google
-Cloud Tasks service in the Google Cloud.
+Example Airflow DAG that createsand deletes Queues and creates, gets, lists,
+runs and deletes Tasks in the Google Cloud Tasks service in the Google Cloud.
"""
import os
from datetime import datetime, timedelta
@@ -27,32 +26,30 @@ from datetime import datetime, timedelta
from google.api_core.retry import Retry
from google.cloud.tasks_v2.types import Queue
from google.protobuf import timestamp_pb2
-from google.protobuf.field_mask_pb2 import FieldMask
from airflow import models
+from airflow.decorators import task
from airflow.models.baseoperator import chain
-from airflow.operators.bash import BashOperator
from airflow.providers.google.cloud.operators.tasks import (
CloudTasksQueueCreateOperator,
CloudTasksQueueDeleteOperator,
- CloudTasksQueueGetOperator,
- CloudTasksQueuePauseOperator,
- CloudTasksQueuePurgeOperator,
- CloudTasksQueueResumeOperator,
- CloudTasksQueuesListOperator,
- CloudTasksQueueUpdateOperator,
CloudTasksTaskCreateOperator,
CloudTasksTaskDeleteOperator,
CloudTasksTaskGetOperator,
CloudTasksTaskRunOperator,
CloudTasksTasksListOperator,
)
+from airflow.utils.trigger_rule import TriggerRule
+
+ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
+DAG_ID = "cloud_tasks_tasks"
timestamp = timestamp_pb2.Timestamp()
timestamp.FromDatetime(datetime.now() + timedelta(hours=12))
-LOCATION = "europe-west1"
-QUEUE_ID = os.environ.get('GCP_TASKS_QUEUE_ID', "cloud-tasks-queue")
+LOCATION = "europe-central2"
+# queue cannot use recent names even if queue was removed
+QUEUE_ID = f"queue-{ENV_ID}-{DAG_ID.replace('_', '-')}"
TASK_NAME = "task-to-run"
@@ -66,104 +63,49 @@ TASK = {
}
with models.DAG(
- "example_gcp_tasks",
- schedule_interval='@once', # Override to match your needs
+ dag_id=DAG_ID,
+ schedule_interval='@once',
start_date=datetime(2021, 1, 1),
catchup=False,
- tags=['example'],
+ tags=['example', "tasks"],
) as dag:
- # Queue operations
- # [START create_queue]
+ @task(task_id="random_string")
+ def generate_random_string():
+ """
+ Generate random string for queue and task names.
+ Queue name cannot be repeated in preceding 7 days and
+ task name in the last 1 hour.
+ """
+ import random
+ import string
+
+ return "".join(random.choices(string.ascii_uppercase + string.digits, k=8))
+
+ random_string = generate_random_string()
+
create_queue = CloudTasksQueueCreateOperator(
location=LOCATION,
task_queue=Queue(stackdriver_logging_config=dict(sampling_ratio=0.5)),
- queue_name=QUEUE_ID,
+ queue_name=QUEUE_ID + "{{ task_instance.xcom_pull(task_ids='random_string') }}",
retry=Retry(maximum=10.0),
timeout=5,
task_id="create_queue",
)
- # [END create_queue]
- # [START delete_queue]
delete_queue = CloudTasksQueueDeleteOperator(
location=LOCATION,
- queue_name=QUEUE_ID,
+ queue_name=QUEUE_ID + "{{ task_instance.xcom_pull(task_ids='random_string') }}",
task_id="delete_queue",
)
- # [END delete_queue]
-
- # [START resume_queue]
- resume_queue = CloudTasksQueueResumeOperator(
- location=LOCATION,
- queue_name=QUEUE_ID,
- task_id="resume_queue",
- )
- # [END resume_queue]
-
- # [START pause_queue]
- pause_queue = CloudTasksQueuePauseOperator(
- location=LOCATION,
- queue_name=QUEUE_ID,
- task_id="pause_queue",
- )
- # [END pause_queue]
-
- # [START purge_queue]
- purge_queue = CloudTasksQueuePurgeOperator(
- location=LOCATION,
- queue_name=QUEUE_ID,
- task_id="purge_queue",
- )
- # [END purge_queue]
-
- # [START get_queue]
- get_queue = CloudTasksQueueGetOperator(
- location=LOCATION,
- queue_name=QUEUE_ID,
- task_id="get_queue",
- )
-
- get_queue_result = BashOperator(
- task_id="get_queue_result",
- bash_command=f"echo {get_queue.output}",
- )
- # [END get_queue]
+ delete_queue.trigger_rule = TriggerRule.ALL_DONE
- get_queue >> get_queue_result
-
- # [START update_queue]
- update_queue = CloudTasksQueueUpdateOperator(
- task_queue=Queue(stackdriver_logging_config=dict(sampling_ratio=1)),
- location=LOCATION,
- queue_name=QUEUE_ID,
- update_mask=FieldMask(paths=["stackdriver_logging_config.sampling_ratio"]),
- task_id="update_queue",
- )
- # [END update_queue]
-
- # [START list_queue]
- list_queue = CloudTasksQueuesListOperator(location=LOCATION, task_id="list_queue")
- # [END list_queue]
-
- chain(
- create_queue,
- update_queue,
- pause_queue,
- resume_queue,
- purge_queue,
- get_queue,
- list_queue,
- delete_queue,
- )
-
- # Tasks operations
# [START create_task]
create_task = CloudTasksTaskCreateOperator(
location=LOCATION,
- queue_name=QUEUE_ID,
+ queue_name=QUEUE_ID + "{{ task_instance.xcom_pull(task_ids='random_string') }}",
task=TASK,
- task_name=TASK_NAME,
+ task_name=TASK_NAME + "{{ task_instance.xcom_pull(task_ids='random_string') }}",
retry=Retry(maximum=10.0),
timeout=5,
task_id="create_task_to_run",
@@ -173,8 +115,8 @@ with models.DAG(
# [START tasks_get]
tasks_get = CloudTasksTaskGetOperator(
location=LOCATION,
- queue_name=QUEUE_ID,
- task_name=TASK_NAME,
+ queue_name=QUEUE_ID + "{{ task_instance.xcom_pull(task_ids='random_string') }}",
+ task_name=TASK_NAME + "{{ task_instance.xcom_pull(task_ids='random_string') }}",
task_id="tasks_get",
)
# [END tasks_get]
@@ -182,20 +124,41 @@ with models.DAG(
# [START run_task]
run_task = CloudTasksTaskRunOperator(
location=LOCATION,
- queue_name=QUEUE_ID,
- task_name=TASK_NAME,
+ queue_name=QUEUE_ID + "{{ task_instance.xcom_pull(task_ids='random_string') }}",
+ task_name=TASK_NAME + "{{ task_instance.xcom_pull(task_ids='random_string') }}",
task_id="run_task",
)
# [END run_task]
# [START list_tasks]
- list_tasks = CloudTasksTasksListOperator(location=LOCATION, queue_name=QUEUE_ID, task_id="list_tasks")
+ list_tasks = CloudTasksTasksListOperator(
+ location=LOCATION,
+ queue_name=QUEUE_ID + "{{ task_instance.xcom_pull(task_ids='random_string') }}",
+ task_id="list_tasks",
+ )
# [END list_tasks]
# [START delete_task]
delete_task = CloudTasksTaskDeleteOperator(
- location=LOCATION, queue_name=QUEUE_ID, task_name=TASK_NAME, task_id="delete_task"
+ location=LOCATION,
+ queue_name=QUEUE_ID + "{{ task_instance.xcom_pull(task_ids='random_string') }}",
+ task_name=TASK_NAME + "{{ task_instance.xcom_pull(task_ids='random_string') }}",
+ task_id="delete_task",
)
# [END delete_task]
- chain(purge_queue, create_task, tasks_get, list_tasks, run_task, delete_task, delete_queue)
+ chain(
+ random_string, create_queue, create_task, tasks_get, list_tasks, run_task, delete_task, delete_queue
+ )
+
+ from tests.system.utils.watcher import watcher
+
+ # This test needs watcher in order to properly mark success/failure
+ # when "tearDown" task with trigger rule is part of the DAG
+ list(dag.tasks) >> watcher()
+
+
+from tests.system.utils import get_test_run # noqa: E402
+
+# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest)
+test_run = get_test_run(dag)