You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2022/09/10 10:03:49 UTC
[airflow] branch main updated: Cloud Video Intelligence Operators assets & system tests migration (AIP-47) (#26132)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 23ad7e292a Cloud Video Intelligence Operators assets & system tests migration (AIP-47) (#26132)
23ad7e292a is described below
commit 23ad7e292a29e43436170078ae132569f8fbf1e4
Author: Beata Kossakowska <10...@users.noreply.github.com>
AuthorDate: Sat Sep 10 12:03:39 2022 +0200
Cloud Video Intelligence Operators assets & system tests migration (AIP-47) (#26132)
Cloud Video Intelligence Operators assets & system tests migration (AIP-47)
---
.../operators/cloud/video_intelligence.rst | 18 +++---
.../operators/test_video_intelligence_system.py | 49 ----------------
.../google/cloud/video_intelligence/__init__.py | 16 ++++++
.../example_video_intelligence.py | 66 +++++++++++++++++++---
4 files changed, 83 insertions(+), 66 deletions(-)
diff --git a/docs/apache-airflow-providers-google/operators/cloud/video_intelligence.rst b/docs/apache-airflow-providers-google/operators/cloud/video_intelligence.rst
index 2b5d441234..8b3f540a9f 100644
--- a/docs/apache-airflow-providers-google/operators/cloud/video_intelligence.rst
+++ b/docs/apache-airflow-providers-google/operators/cloud/video_intelligence.rst
@@ -38,12 +38,12 @@ Using the operator
Input uri is an uri to a file in Google Cloud Storage
-.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_video_intelligence.py
+.. exampleinclude:: /../../tests/system/providers/google/cloud/video_intelligence/example_video_intelligence.py
:language: python
:start-after: [START howto_operator_video_intelligence_other_args]
:end-before: [END howto_operator_video_intelligence_other_args]
-.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_video_intelligence.py
+.. exampleinclude:: /../../tests/system/providers/google/cloud/video_intelligence/example_video_intelligence.py
:language: python
:dedent: 4
:start-after: [START howto_operator_video_intelligence_detect_labels]
@@ -51,7 +51,7 @@ Input uri is an uri to a file in Google Cloud Storage
You can use the annotation output via Xcom:
-.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_video_intelligence.py
+.. exampleinclude:: /../../tests/system/providers/google/cloud/video_intelligence/example_video_intelligence.py
:language: python
:dedent: 4
:start-after: [START howto_operator_video_intelligence_detect_labels_result]
@@ -87,7 +87,7 @@ Arguments
Input uri is an uri to a file in Google Cloud Storage
-.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_video_intelligence.py
+.. exampleinclude:: /../../tests/system/providers/google/cloud/video_intelligence/example_video_intelligence.py
:language: python
:start-after: [START howto_operator_video_intelligence_other_args]
:end-before: [END howto_operator_video_intelligence_other_args]
@@ -95,7 +95,7 @@ Input uri is an uri to a file in Google Cloud Storage
Using the operator
""""""""""""""""""
-.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_video_intelligence.py
+.. exampleinclude:: /../../tests/system/providers/google/cloud/video_intelligence/example_video_intelligence.py
:language: python
:dedent: 4
:start-after: [START howto_operator_video_intelligence_detect_explicit_content]
@@ -103,7 +103,7 @@ Using the operator
You can use the annotation output via Xcom:
-.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_video_intelligence.py
+.. exampleinclude:: /../../tests/system/providers/google/cloud/video_intelligence/example_video_intelligence.py
:language: python
:dedent: 4
:start-after: [START howto_operator_video_intelligence_detect_explicit_content_result]
@@ -139,7 +139,7 @@ Arguments
Input uri is an uri to a file in Google Cloud Storage
-.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_video_intelligence.py
+.. exampleinclude:: /../../tests/system/providers/google/cloud/video_intelligence/example_video_intelligence.py
:language: python
:start-after: [START howto_operator_video_intelligence_other_args]
:end-before: [END howto_operator_video_intelligence_other_args]
@@ -147,7 +147,7 @@ Input uri is an uri to a file in Google Cloud Storage
Using the operator
""""""""""""""""""
-.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_video_intelligence.py
+.. exampleinclude:: /../../tests/system/providers/google/cloud/video_intelligence/example_video_intelligence.py
:language: python
:dedent: 4
:start-after: [START howto_operator_video_intelligence_detect_video_shots]
@@ -155,7 +155,7 @@ Using the operator
You can use the annotation output via Xcom:
-.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_video_intelligence.py
+.. exampleinclude:: /../../tests/system/providers/google/cloud/video_intelligence/example_video_intelligence.py
:language: python
:dedent: 4
:start-after: [START howto_operator_video_intelligence_detect_video_shots_result]
diff --git a/tests/providers/google/cloud/operators/test_video_intelligence_system.py b/tests/providers/google/cloud/operators/test_video_intelligence_system.py
deleted file mode 100644
index 8c92bf1e12..0000000000
--- a/tests/providers/google/cloud/operators/test_video_intelligence_system.py
+++ /dev/null
@@ -1,49 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-import os
-
-import pytest
-
-from airflow.providers.google.cloud.example_dags.example_video_intelligence import GCP_BUCKET_NAME
-from tests.providers.google.cloud.utils.gcp_authenticator import GCP_AI_KEY, GCP_GCS_KEY
-from tests.test_utils.gcp_system_helpers import CLOUD_DAG_FOLDER, GoogleSystemTest, provide_gcp_context
-
-GCP_PROJECT_ID = os.environ.get("GCP_PROJECT_ID", "example-project")
-GCP_VIDEO_SOURCE_URL = "https://www.sample-videos.com/video123/mp4/720/big_buck_bunny_720p_1mb.mp4"
-
-
-@pytest.mark.backend("mysql", "postgres")
-@pytest.mark.credential_file(GCP_AI_KEY)
-class CloudVideoIntelligenceExampleDagsTest(GoogleSystemTest):
- @provide_gcp_context(GCP_AI_KEY)
- def setUp(self):
- super().setUp()
- self.create_gcs_bucket(GCP_BUCKET_NAME, location="europe-north1")
- self.execute_with_ctx(
- cmd=["bash", "-c", f"curl {GCP_VIDEO_SOURCE_URL} | gsutil cp - gs://{GCP_BUCKET_NAME}/video.mp4"],
- key=GCP_GCS_KEY,
- )
-
- @provide_gcp_context(GCP_AI_KEY)
- def tearDown(self):
- self.delete_gcs_bucket(GCP_BUCKET_NAME)
- super().tearDown()
-
- @provide_gcp_context(GCP_AI_KEY)
- def test_example_dag(self):
- self.run_dag('example_gcp_video_intelligence', CLOUD_DAG_FOLDER)
diff --git a/tests/system/providers/google/cloud/video_intelligence/__init__.py b/tests/system/providers/google/cloud/video_intelligence/__init__.py
new file mode 100644
index 0000000000..13a83393a9
--- /dev/null
+++ b/tests/system/providers/google/cloud/video_intelligence/__init__.py
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
diff --git a/airflow/providers/google/cloud/example_dags/example_video_intelligence.py b/tests/system/providers/google/cloud/video_intelligence/example_video_intelligence.py
similarity index 67%
rename from airflow/providers/google/cloud/example_dags/example_video_intelligence.py
rename to tests/system/providers/google/cloud/video_intelligence/example_video_intelligence.py
index 9e8183e785..9a214e3d28 100644
--- a/airflow/providers/google/cloud/example_dags/example_video_intelligence.py
+++ b/tests/system/providers/google/cloud/video_intelligence/example_video_intelligence.py
@@ -22,7 +22,7 @@ Cloud Platform.
This DAG relies on the following OS environment variables:
-* GCP_BUCKET_NAME - Google Cloud Storage bucket where the file exists.
+* BUCKET_NAME - Google Cloud Storage bucket where the file exists.
"""
import os
from datetime import datetime
@@ -30,30 +30,53 @@ from datetime import datetime
from google.api_core.retry import Retry
from airflow import models
+from airflow.models.baseoperator import chain
from airflow.operators.bash import BashOperator
+from airflow.providers.google.cloud.operators.gcs import GCSCreateBucketOperator, GCSDeleteBucketOperator
from airflow.providers.google.cloud.operators.video_intelligence import (
CloudVideoIntelligenceDetectVideoExplicitContentOperator,
CloudVideoIntelligenceDetectVideoLabelsOperator,
CloudVideoIntelligenceDetectVideoShotsOperator,
)
+from airflow.providers.google.cloud.transfers.gcs_to_gcs import GCSToGCSOperator
+from airflow.utils.trigger_rule import TriggerRule
+
+ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
+
+DAG_ID = "example_gcp_video_intelligence"
+
+# Public bucket holding the sample data
+BUCKET_NAME_SRC = "cloud-samples-data"
+# Path to the data inside the public bucket
+PATH_SRC = "video/cat.mp4"
# [START howto_operator_video_intelligence_os_args]
-GCP_BUCKET_NAME = os.environ.get("GCP_VIDEO_INTELLIGENCE_BUCKET_NAME", "INVALID BUCKET NAME")
+BUCKET_NAME_DST = f"bucket-src-{DAG_ID}-{ENV_ID}"
# [END howto_operator_video_intelligence_os_args]
+FILE_NAME = "video.mp4"
# [START howto_operator_video_intelligence_other_args]
-INPUT_URI = f"gs://{GCP_BUCKET_NAME}/video.mp4"
+INPUT_URI = f"gs://{BUCKET_NAME_DST}/{FILE_NAME}"
# [END howto_operator_video_intelligence_other_args]
-
with models.DAG(
- "example_gcp_video_intelligence",
+ DAG_ID,
start_date=datetime(2021, 1, 1),
catchup=False,
tags=['example'],
) as dag:
+ create_bucket = GCSCreateBucketOperator(task_id="create_bucket", bucket_name=BUCKET_NAME_DST)
+
+ copy_single_file = GCSToGCSOperator(
+ task_id="copy_single_gcs_file",
+ source_bucket=BUCKET_NAME_SRC,
+ source_object=PATH_SRC,
+ destination_bucket=BUCKET_NAME_DST,
+ destination_object=FILE_NAME,
+ )
+
# [START howto_operator_video_intelligence_detect_labels]
detect_video_label = CloudVideoIntelligenceDetectVideoLabelsOperator(
input_uri=INPUT_URI,
@@ -110,6 +133,33 @@ with models.DAG(
)
# [END howto_operator_video_intelligence_detect_video_shots_result]
- detect_video_label >> detect_video_label_result
- detect_video_explicit_content >> detect_video_explicit_content_result
- detect_video_shots >> detect_video_shots_result
+ delete_bucket = GCSDeleteBucketOperator(
+ task_id="delete_bucket", bucket_name=BUCKET_NAME_DST, trigger_rule=TriggerRule.ALL_DONE
+ )
+
+ chain(
+ # TEST SETUP
+ create_bucket,
+ copy_single_file,
+ # TEST BODY
+ detect_video_label,
+ detect_video_label_result,
+ detect_video_explicit_content,
+ detect_video_explicit_content_result,
+ detect_video_shots,
+ detect_video_shots_result,
+ # TEST TEARDOWN
+ delete_bucket,
+ )
+
+ from tests.system.utils.watcher import watcher
+
+ # This test needs watcher in order to properly mark success/failure
+ # when "tearDown" task with trigger rule is part of the DAG
+ list(dag.tasks) >> watcher()
+
+
+from tests.system.utils import get_test_run # noqa: E402
+
+# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest)
+test_run = get_test_run(dag)