You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2022/07/12 11:58:51 UTC
[airflow] branch main updated: Migrate Google sheets example to new design AIP-47 (#24975)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new a13c51e224 Migrate Google sheets example to new design AIP-47 (#24975)
a13c51e224 is described below
commit a13c51e224e6299387d8b704e12cb7b4c8f61718
Author: Chenglong Yan <al...@gmail.com>
AuthorDate: Tue Jul 12 19:58:44 2022 +0800
Migrate Google sheets example to new design AIP-47 (#24975)
related: #22447, #22430
---
.../operators/suite/sheets.rst | 4 +-
.../google/suite/operators/test_sheets_system.py | 50 ----------------------
.../providers/google/cloud/gcs}/example_sheets.py | 48 ++++++++++++++++++---
3 files changed, 43 insertions(+), 59 deletions(-)
diff --git a/docs/apache-airflow-providers-google/operators/suite/sheets.rst b/docs/apache-airflow-providers-google/operators/suite/sheets.rst
index 3ef6489b52..29c4f8de4e 100644
--- a/docs/apache-airflow-providers-google/operators/suite/sheets.rst
+++ b/docs/apache-airflow-providers-google/operators/suite/sheets.rst
@@ -44,7 +44,7 @@ Create spreadsheet
To create new spreadsheet you can use the
:class:`~airflow.providers.google.suite.operators.sheets.GoogleSheetsCreateSpreadsheetOperator`.
-.. exampleinclude:: /../../airflow/providers/google/suite/example_dags/example_sheets.py
+.. exampleinclude:: /../../tests/system/providers/google/cloud/gcs/example_sheets.py
:language: python
:dedent: 4
:start-after: [START create_spreadsheet]
@@ -55,7 +55,7 @@ You can use :ref:`Jinja templating <concepts:jinja-templating>` with
To get the URL of newly created spreadsheet use XCom value:
-.. exampleinclude:: /../../airflow/providers/google/suite/example_dags/example_sheets.py
+.. exampleinclude:: /../../tests/system/providers/google/cloud/gcs/example_sheets.py
:language: python
:dedent: 4
:start-after: [START print_spreadsheet_url]
diff --git a/tests/providers/google/suite/operators/test_sheets_system.py b/tests/providers/google/suite/operators/test_sheets_system.py
deleted file mode 100644
index b8b341bce4..0000000000
--- a/tests/providers/google/suite/operators/test_sheets_system.py
+++ /dev/null
@@ -1,50 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-import pytest
-
-from airflow.providers.google.suite.example_dags.example_sheets import GCS_BUCKET
-from tests.providers.google.cloud.utils.gcp_authenticator import GCP_GCS_KEY
-from tests.test_utils.gcp_system_helpers import GSUITE_DAG_FOLDER, GoogleSystemTest, provide_gcp_context
-
-# Required scopes
-SCOPES = [
- 'https://www.googleapis.com/auth/drive',
- 'https://www.googleapis.com/auth/cloud-platform',
-]
-
-
-@pytest.fixture
-def helper():
- GoogleSystemTest.create_gcs_bucket(GCS_BUCKET)
- yield
- GoogleSystemTest.delete_gcs_bucket(GCS_BUCKET)
-
-
-@pytest.mark.system("google.suite")
-@pytest.mark.credential_file(GCP_GCS_KEY)
-class TestSystemSheetsToGcs(GoogleSystemTest):
- def setUp(self):
- super().setUp()
-
- @pytest.mark.usefixtures("helper")
- @provide_gcp_context(GCP_GCS_KEY, scopes=SCOPES)
- def test_run_example_dag(self):
- self.run_dag('example_sheets_gcs', GSUITE_DAG_FOLDER)
-
- def tearDown(self):
- super().tearDown()
diff --git a/airflow/providers/google/suite/example_dags/example_sheets.py b/tests/system/providers/google/cloud/gcs/example_sheets.py
similarity index 64%
rename from airflow/providers/google/suite/example_dags/example_sheets.py
rename to tests/system/providers/google/cloud/gcs/example_sheets.py
index e5b3f820b2..c17543d4f8 100644
--- a/airflow/providers/google/suite/example_dags/example_sheets.py
+++ b/tests/system/providers/google/cloud/gcs/example_sheets.py
@@ -21,11 +21,17 @@ from datetime import datetime
from airflow import models
from airflow.operators.bash import BashOperator
+from airflow.providers.google.cloud.operators.gcs import GCSCreateBucketOperator, GCSDeleteBucketOperator
from airflow.providers.google.cloud.transfers.sheets_to_gcs import GoogleSheetsToGCSOperator
from airflow.providers.google.suite.operators.sheets import GoogleSheetsCreateSpreadsheetOperator
from airflow.providers.google.suite.transfers.gcs_to_sheets import GCSToGoogleSheetsOperator
+from airflow.utils.trigger_rule import TriggerRule
-GCS_BUCKET = os.environ.get("SHEETS_GCS_BUCKET", "test28397ye")
+ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
+PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT")
+DAG_ID = "example_sheets_gcs"
+
+BUCKET_NAME = f"bucket_{DAG_ID}_{ENV_ID}"
SPREADSHEET_ID = os.environ.get("SPREADSHEET_ID", "1234567890qwerty")
NEW_SPREADSHEET_ID = os.environ.get("NEW_SPREADSHEET_ID", "1234567890qwerty")
@@ -35,16 +41,20 @@ SPREADSHEET = {
}
with models.DAG(
- "example_sheets_gcs",
+ DAG_ID,
schedule_interval='@once', # Override to match your needs,
start_date=datetime(2021, 1, 1),
catchup=False,
- tags=["example"],
+ tags=["example", "sheets"],
) as dag:
+ create_bucket = GCSCreateBucketOperator(
+ task_id="create_bucket", bucket_name=BUCKET_NAME, project_id=PROJECT_ID
+ )
+
# [START upload_sheet_to_gcs]
upload_sheet_to_gcs = GoogleSheetsToGCSOperator(
task_id="upload_sheet_to_gcs",
- destination_bucket=GCS_BUCKET,
+ destination_bucket=BUCKET_NAME,
spreadsheet_id=SPREADSHEET_ID,
)
# [END upload_sheet_to_gcs]
@@ -65,11 +75,35 @@ with models.DAG(
# [START upload_gcs_to_sheet]
upload_gcs_to_sheet = GCSToGoogleSheetsOperator(
task_id="upload_gcs_to_sheet",
- bucket_name=GCS_BUCKET,
+ bucket_name=BUCKET_NAME,
object_name="{{ task_instance.xcom_pull('upload_sheet_to_gcs')[0] }}",
spreadsheet_id=NEW_SPREADSHEET_ID,
)
# [END upload_gcs_to_sheet]
- create_spreadsheet >> print_spreadsheet_url
- upload_sheet_to_gcs >> upload_gcs_to_sheet
+ delete_bucket = GCSDeleteBucketOperator(
+ task_id="delete_bucket", bucket_name=BUCKET_NAME, trigger_rule=TriggerRule.ALL_DONE
+ )
+
+ (
+ # TEST SETUP
+ create_bucket
+ # TEST BODY
+ >> create_spreadsheet
+ >> print_spreadsheet_url
+ >> upload_sheet_to_gcs
+ >> upload_gcs_to_sheet
+ # TEST TEARDOWN
+ >> delete_bucket
+ )
+
+ from tests.system.utils.watcher import watcher
+
+ # This test needs watcher in order to properly mark success/failure
+ # when "tearDown" task with trigger rule is part of the DAG
+ list(dag.tasks) >> watcher()
+
+from tests.system.utils import get_test_run # noqa: E402
+
+# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest)
+test_run = get_test_run(dag)