You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2022/04/13 19:30:12 UTC
[airflow] branch main updated: migrate system test gcs_to_bigquery into new design (#22753)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 9a623e94cb migrate system test gcs_to_bigquery into new design (#22753)
9a623e94cb is described below
commit 9a623e94cb3e4f02cbe566e02f75f4a894edc60d
Author: Joppe Vos <44...@users.noreply.github.com>
AuthorDate: Wed Apr 13 21:30:05 2022 +0200
migrate system test gcs_to_bigquery into new design (#22753)
---
.../operators/cloud/gcs.rst | 2 +-
.../cloud/transfers/test_gcs_to_bigquery_system.py | 36 --------------------
.../google/gcs}/example_gcs_to_bigquery.py | 38 ++++++++++++++++++----
3 files changed, 32 insertions(+), 44 deletions(-)
diff --git a/docs/apache-airflow-providers-google/operators/cloud/gcs.rst b/docs/apache-airflow-providers-google/operators/cloud/gcs.rst
index 97703adc7c..54a7d4ba78 100644
--- a/docs/apache-airflow-providers-google/operators/cloud/gcs.rst
+++ b/docs/apache-airflow-providers-google/operators/cloud/gcs.rst
@@ -41,7 +41,7 @@ Use the
:class:`~airflow.providers.google.cloud.transfers.gcs_to_bigquery.GCSToBigQueryOperator`
to execute a BigQuery load job.
-.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_gcs_to_bigquery.py
+.. exampleinclude:: /../../tests/system/providers/google/gcs/example_gcs_to_bigquery.py
:language: python
:dedent: 4
:start-after: [START howto_operator_gcs_to_bigquery]
diff --git a/tests/providers/google/cloud/transfers/test_gcs_to_bigquery_system.py b/tests/providers/google/cloud/transfers/test_gcs_to_bigquery_system.py
deleted file mode 100644
index 5971e94ba6..0000000000
--- a/tests/providers/google/cloud/transfers/test_gcs_to_bigquery_system.py
+++ /dev/null
@@ -1,36 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-import pytest
-
-from tests.providers.google.cloud.utils.gcp_authenticator import GCP_BIGQUERY_KEY
-from tests.test_utils.gcp_system_helpers import CLOUD_DAG_FOLDER, GoogleSystemTest, provide_gcp_context
-
-
-@pytest.mark.backend("mysql", "postgres")
-@pytest.mark.credential_file(GCP_BIGQUERY_KEY)
-class TestGoogleCloudStorageToBigQueryExample(GoogleSystemTest):
- def setUp(self):
- super().setUp()
-
- @provide_gcp_context(GCP_BIGQUERY_KEY)
- def test_run_example_dag_gcs_to_bigquery_operator(self):
- self.run_dag('example_gcs_to_bigquery_operator', CLOUD_DAG_FOLDER)
-
- def tearDown(self):
- super().tearDown()
diff --git a/airflow/providers/google/cloud/example_dags/example_gcs_to_bigquery.py b/tests/system/providers/google/gcs/example_gcs_to_bigquery.py
similarity index 69%
rename from airflow/providers/google/cloud/example_dags/example_gcs_to_bigquery.py
rename to tests/system/providers/google/gcs/example_gcs_to_bigquery.py
index 454320e4a4..52822e38eb 100644
--- a/airflow/providers/google/cloud/example_dags/example_gcs_to_bigquery.py
+++ b/tests/system/providers/google/gcs/example_gcs_to_bigquery.py
@@ -29,19 +29,24 @@ from airflow.providers.google.cloud.operators.bigquery import (
BigQueryDeleteDatasetOperator,
)
from airflow.providers.google.cloud.transfers.gcs_to_bigquery import GCSToBigQueryOperator
+from airflow.utils.trigger_rule import TriggerRule
-DATASET_NAME = os.environ.get("GCP_DATASET_NAME", 'airflow_test')
-TABLE_NAME = os.environ.get("GCP_TABLE_NAME", 'gcs_to_bq_table')
+ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
+DAG_ID = "gcs_to_bigquery_operator"
+
+DATASET_NAME = f"dataset_{DAG_ID}_{ENV_ID}"
+TABLE_NAME = "test"
+PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT")
with models.DAG(
- dag_id='example_gcs_to_bigquery_operator',
+ dag_id=DAG_ID,
+ schedule_interval='@once',
start_date=datetime(2021, 1, 1),
catchup=False,
- schedule_interval='@once',
- tags=['example'],
+ tags=['example', "gcs"],
) as dag:
create_test_dataset = BigQueryCreateEmptyDatasetOperator(
- task_id='create_airflow_test_dataset', dataset_id=DATASET_NAME
+ task_id='create_airflow_test_dataset', dataset_id=DATASET_NAME, project_id=PROJECT_ID
)
# [START howto_operator_gcs_to_bigquery]
@@ -62,6 +67,25 @@ with models.DAG(
task_id='delete_airflow_test_dataset',
dataset_id=DATASET_NAME,
delete_contents=True,
+ trigger_rule=TriggerRule.ALL_DONE,
+ )
+
+ (
+ # TEST SETUP
+ create_test_dataset
+ # TEST BODY
+ >> load_csv
+ # TEST TEARDOWN
+ >> delete_test_dataset
)
-create_test_dataset >> load_csv >> delete_test_dataset
+ from tests.system.utils.watcher import watcher
+
+ # This test needs watcher in order to properly mark success/failure
+ # when "tearDown" task with trigger rule is part of the DAG
+ list(dag.tasks) >> watcher()
+
+from tests.system.utils import get_test_run # noqa: E402
+
+# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest)
+test_run = get_test_run(dag)