You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2022/10/10 04:22:07 UTC
[airflow] branch main updated: Local filesystem to Google Drive Operator - system tests migration (AIP-47) (#26797)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new b54a2de8c7 Local filesystem to Google Drive Operator - system tests migration (AIP-47) (#26797)
b54a2de8c7 is described below
commit b54a2de8c74feb1ea215a98ffaddc5c46713c5cb
Author: Beata Kossakowska <10...@users.noreply.github.com>
AuthorDate: Mon Oct 10 06:21:59 2022 +0200
Local filesystem to Google Drive Operator - system tests migration (AIP-47) (#26797)
---
.../example-dags.rst | 3 +-
.../operators/transfer/local_to_drive.rst | 8 +++++
.../google/suite}/example_local_to_drive.py | 39 +++++++++++++++++++---
.../providers/google/suite/resources}/__init__.py | 1 -
.../system/providers/google/suite/resources/test1 | 0
.../system/providers/google/suite/resources/test2 | 0
6 files changed, 43 insertions(+), 8 deletions(-)
diff --git a/docs/apache-airflow-providers-google/example-dags.rst b/docs/apache-airflow-providers-google/example-dags.rst
index c4c05792a4..b912ef3ea4 100644
--- a/docs/apache-airflow-providers-google/example-dags.rst
+++ b/docs/apache-airflow-providers-google/example-dags.rst
@@ -17,7 +17,6 @@
Example DAGs
============
-
You can learn how to use Google integrations by analyzing the source code of the example DAGs:
* `Google Ads <https://github.com/apache/airflow/tree/providers-google/8.1.0/tests/system/providers/google/ads>`__
@@ -25,5 +24,5 @@ You can learn how to use Google integrations by analyzing the source code of the
* `Google Cloud <https://github.com/apache/airflow/tree/providers-google/8.0.0/tests/system/providers/google>`__
* `Google Firebase <https://github.com/apache/airflow/tree/providers-google/8.1.0/tests/system/providers/google/firebase>`__
* `Google Marketing Platform <https://github.com/apache/airflow/tree/providers-google/8.0.0/airflow/providers/google/marketing_platform/example_dags>`__
-* `Google Workplace <https://github.com/apache/airflow/tree/providers-google/8.0.0/airflow/providers/google/suite/example_dags>`__ (formerly Google Suite)
+* `Google Workplace <https://github.com/apache/airflow/tree/providers-google/8.3.0/tests/system/providers/google/suite>`__ (formerly Google Suite)
* `Google LevelDB <https://github.com/apache/airflow/tree/providers-google/8.0.0/tests/system/providers/google/leveldb>`__
diff --git a/docs/apache-airflow-providers-google/operators/transfer/local_to_drive.rst b/docs/apache-airflow-providers-google/operators/transfer/local_to_drive.rst
index f9b77db3b2..86205a9faa 100644
--- a/docs/apache-airflow-providers-google/operators/transfer/local_to_drive.rst
+++ b/docs/apache-airflow-providers-google/operators/transfer/local_to_drive.rst
@@ -35,3 +35,11 @@ LocalFilesystemToGoogleDriveOperator
data from local filesystem to GoogleDrive.
When you use this operator, you can upload a list of files.
+
+Below is an example of using this operator to upload data from local filesystem to Google Drive.
+
+.. exampleinclude:: /../../tests/system/providers/google/suite/example_local_to_drive.py
+ :language: python
+ :dedent: 0
+ :start-after: [START howto_operator_local_to_drive_upload_single_file]
+ :end-before: [END howto_operator_local_to_drive_upload_single_file]
diff --git a/airflow/providers/google/suite/example_dags/example_local_to_drive.py b/tests/system/providers/google/suite/example_local_to_drive.py
similarity index 65%
rename from airflow/providers/google/suite/example_dags/example_local_to_drive.py
rename to tests/system/providers/google/suite/example_local_to_drive.py
index 5c8852d6e3..5a1f44e3fd 100644
--- a/airflow/providers/google/suite/example_dags/example_local_to_drive.py
+++ b/tests/system/providers/google/suite/example_local_to_drive.py
@@ -17,6 +17,9 @@
# under the License.
"""
Example DAG using LocalFilesystemToGoogleDriveOperator.
+
+Using this operator requires the following additional scopes:
+https://www.googleapis.com/auth/drive
"""
from __future__ import annotations
@@ -26,16 +29,27 @@ from pathlib import Path
from airflow import models
from airflow.providers.google.suite.transfers.local_to_drive import LocalFilesystemToGoogleDriveOperator
-SINGLE_FILE_LOCAL_PATHS = [Path("test1")]
-MULTIPLE_FILES_LOCAL_PATHS = [Path("test1"), Path("test2")]
-DRIVE_FOLDER = Path("test-folder")
+DAG_ID = "example_local_to_drive"
+
+FILE_NAME_1 = "test1"
+FILE_NAME_2 = "test2"
+
+LOCAL_PATH = str(Path(__file__).parent / "resources")
+
+SINGLE_FILE_LOCAL_PATHS = [str(Path(LOCAL_PATH) / FILE_NAME_1)]
+MULTIPLE_FILES_LOCAL_PATHS = [str(Path(LOCAL_PATH) / FILE_NAME_1), str(Path(LOCAL_PATH) / FILE_NAME_2)]
+
+DRIVE_FOLDER = "test-folder"
+
with models.DAG(
- "example_local_to_drive",
+ DAG_ID,
+ schedule="@once",
start_date=datetime(2021, 1, 1),
catchup=False,
tags=["example"],
) as dag:
+
# [START howto_operator_local_to_drive_upload_single_file]
upload_single_file = LocalFilesystemToGoogleDriveOperator(
task_id="upload_single_file",
@@ -53,4 +67,19 @@ with models.DAG(
)
# [END howto_operator_local_to_drive_upload_multiple_files]
- upload_single_file >> upload_multiple_files
+ (
+ # TEST BODY
+ upload_single_file
+ >> upload_multiple_files
+ )
+
+ from tests.system.utils.watcher import watcher
+
+ # This test needs watcher in order to properly mark success/failure
+ # when "tearDown" task with trigger rule is part of the DAG
+ list(dag.tasks) >> watcher()
+
+from tests.system.utils import get_test_run # noqa: E402
+
+# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest)
+test_run = get_test_run(dag)
diff --git a/airflow/providers/google/suite/example_dags/__init__.py b/tests/system/providers/google/suite/resources/__init__.py
similarity index 99%
rename from airflow/providers/google/suite/example_dags/__init__.py
rename to tests/system/providers/google/suite/resources/__init__.py
index 217e5db960..13a83393a9 100644
--- a/airflow/providers/google/suite/example_dags/__init__.py
+++ b/tests/system/providers/google/suite/resources/__init__.py
@@ -1,4 +1,3 @@
-#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
diff --git a/tests/system/providers/google/suite/resources/test1 b/tests/system/providers/google/suite/resources/test1
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/tests/system/providers/google/suite/resources/test2 b/tests/system/providers/google/suite/resources/test2
new file mode 100644
index 0000000000..e69de29bb2