You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2022/07/26 10:44:08 UTC
[airflow] branch main updated: Delete redundant system test bigquery_to_bigquery (#25261)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new b288bf4da4 Delete redundant system test bigquery_to_bigquery (#25261)
b288bf4da4 is described below
commit b288bf4da44301d91ae62131976eb2925decd0aa
Author: Chenglong Yan <al...@gmail.com>
AuthorDate: Tue Jul 26 18:43:56 2022 +0800
Delete redundant system test bigquery_to_bigquery (#25261)
related: #22447, #22430
---
.../example_dags/example_bigquery_to_bigquery.py | 69 ----------------------
.../transfers/test_bigquery_to_bigquery_system.py | 37 ------------
2 files changed, 106 deletions(-)
diff --git a/airflow/providers/google/cloud/example_dags/example_bigquery_to_bigquery.py b/airflow/providers/google/cloud/example_dags/example_bigquery_to_bigquery.py
deleted file mode 100644
index 0d6a5bb2d5..0000000000
--- a/airflow/providers/google/cloud/example_dags/example_bigquery_to_bigquery.py
+++ /dev/null
@@ -1,69 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-"""
-Example Airflow DAG for Google BigQuery service.
-"""
-import os
-from datetime import datetime
-
-from airflow import models
-from airflow.providers.google.cloud.operators.bigquery import (
- BigQueryCreateEmptyDatasetOperator,
- BigQueryCreateEmptyTableOperator,
- BigQueryDeleteDatasetOperator,
-)
-from airflow.providers.google.cloud.transfers.bigquery_to_bigquery import BigQueryToBigQueryOperator
-
-PROJECT_ID = os.environ.get("GCP_PROJECT_ID", "example-project")
-DATASET_NAME = os.environ.get("GCP_BIGQUERY_DATASET_NAME", "test_dataset_transfer")
-ORIGIN = "origin"
-TARGET = "target"
-
-with models.DAG(
- "example_bigquery_to_bigquery",
- schedule_interval='@once', # Override to match your needs
- start_date=datetime(2021, 1, 1),
- catchup=False,
- tags=["example"],
-) as dag:
- copy_selected_data = BigQueryToBigQueryOperator(
- task_id="copy_selected_data",
- source_project_dataset_tables=f"{DATASET_NAME}.{ORIGIN}",
- destination_project_dataset_table=f"{DATASET_NAME}.{TARGET}",
- )
-
- create_dataset = BigQueryCreateEmptyDatasetOperator(task_id="create_dataset", dataset_id=DATASET_NAME)
-
- for table in [ORIGIN, TARGET]:
- create_table = BigQueryCreateEmptyTableOperator(
- task_id=f"create_{table}_table",
- dataset_id=DATASET_NAME,
- table_id=table,
- schema_fields=[
- {"name": "emp_name", "type": "STRING", "mode": "REQUIRED"},
- {"name": "salary", "type": "INTEGER", "mode": "NULLABLE"},
- ],
- )
- create_dataset >> create_table >> copy_selected_data
-
- delete_dataset = BigQueryDeleteDatasetOperator(
- task_id="delete_dataset", dataset_id=DATASET_NAME, delete_contents=True
- )
-
- copy_selected_data >> delete_dataset
diff --git a/tests/providers/google/cloud/transfers/test_bigquery_to_bigquery_system.py b/tests/providers/google/cloud/transfers/test_bigquery_to_bigquery_system.py
deleted file mode 100644
index 845c336d98..0000000000
--- a/tests/providers/google/cloud/transfers/test_bigquery_to_bigquery_system.py
+++ /dev/null
@@ -1,37 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-"""System tests for Google Cloud Build operators"""
-import pytest
-
-from tests.providers.google.cloud.utils.gcp_authenticator import GCP_BIGQUERY_KEY
-from tests.test_utils.gcp_system_helpers import CLOUD_DAG_FOLDER, GoogleSystemTest, provide_gcp_context
-
-
-@pytest.mark.backend("mysql", "postgres")
-@pytest.mark.system("google.cloud")
-@pytest.mark.credential_file(GCP_BIGQUERY_KEY)
-class BigQueryToBigQueryExampleDagsSystemTest(GoogleSystemTest):
- def setUp(self):
- super().setUp()
-
- @provide_gcp_context(GCP_BIGQUERY_KEY)
- def test_run_example_dag_queries(self):
- self.run_dag('example_bigquery_to_bigquery', CLOUD_DAG_FOLDER)
-
- def tearDown(self):
- super().tearDown()