You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2022/02/28 00:15:36 UTC

[airflow] branch main updated: Suppress hook warnings from the Bigquery transfers (#20119)

This is an automated email from the ASF dual-hosted git repository.

potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git


The following commit(s) were added to refs/heads/main by this push:
     new 0c55ca2  Suppress hook warnings from the Bigquery transfers (#20119)
0c55ca2 is described below

commit 0c55ca2cda2433ba79b2f4868361576d41d2b41c
Author: Dmytro Kazanzhy <dk...@gmail.com>
AuthorDate: Mon Feb 28 02:14:37 2022 +0200

    Suppress hook warnings from the Bigquery transfers (#20119)
---
 .../google/cloud/transfers/bigquery_to_bigquery.py |  21 ++--
 .../google/cloud/transfers/gcs_to_bigquery.py      | 133 +++++++++++----------
 .../cloud/transfers/test_bigquery_to_bigquery.py   |   2 +-
 .../google/cloud/transfers/test_gcs_to_bigquery.py | 113 +++++++----------
 4 files changed, 130 insertions(+), 139 deletions(-)

diff --git a/airflow/providers/google/cloud/transfers/bigquery_to_bigquery.py b/airflow/providers/google/cloud/transfers/bigquery_to_bigquery.py
index d439062..527033a 100644
--- a/airflow/providers/google/cloud/transfers/bigquery_to_bigquery.py
+++ b/airflow/providers/google/cloud/transfers/bigquery_to_bigquery.py
@@ -127,13 +127,14 @@ class BigQueryToBigQueryOperator(BaseOperator):
             location=self.location,
             impersonation_chain=self.impersonation_chain,
         )
-        conn = hook.get_conn()
-        cursor = conn.cursor()
-        cursor.run_copy(
-            source_project_dataset_tables=self.source_project_dataset_tables,
-            destination_project_dataset_table=self.destination_project_dataset_table,
-            write_disposition=self.write_disposition,
-            create_disposition=self.create_disposition,
-            labels=self.labels,
-            encryption_configuration=self.encryption_configuration,
-        )
+
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore", DeprecationWarning)
+            hook.run_copy(
+                source_project_dataset_tables=self.source_project_dataset_tables,
+                destination_project_dataset_table=self.destination_project_dataset_table,
+                write_disposition=self.write_disposition,
+                create_disposition=self.create_disposition,
+                labels=self.labels,
+                encryption_configuration=self.encryption_configuration,
+            )
diff --git a/airflow/providers/google/cloud/transfers/gcs_to_bigquery.py b/airflow/providers/google/cloud/transfers/gcs_to_bigquery.py
index b5b91d5..a58f01f 100644
--- a/airflow/providers/google/cloud/transfers/gcs_to_bigquery.py
+++ b/airflow/providers/google/cloud/transfers/gcs_to_bigquery.py
@@ -18,6 +18,7 @@
 """This module contains a Google Cloud Storage to BigQuery operator."""
 
 import json
+import warnings
 from typing import TYPE_CHECKING, Optional, Sequence, Union
 
 from airflow.models import BaseOperator
@@ -163,8 +164,9 @@ class GCSToBigQueryOperator(BaseOperator):
         allow_jagged_rows=False,
         encoding="UTF-8",
         max_id_key=None,
-        bigquery_conn_id='google_cloud_default',
-        google_cloud_storage_conn_id='google_cloud_default',
+        gcp_conn_id='google_cloud_default',
+        bigquery_conn_id=None,
+        google_cloud_storage_conn_id=None,
         delegate_to=None,
         schema_update_options=(),
         src_fmt_configs=None,
@@ -179,6 +181,15 @@ class GCSToBigQueryOperator(BaseOperator):
         description=None,
         **kwargs,
     ):
+        # To preserve backward compatibility. Remove one day
+        if bigquery_conn_id or google_cloud_storage_conn_id:
+            warnings.warn(
+                "The bigquery_conn_id and google_cloud_storage_conn_id parameters have been deprecated. "
+                "You should pass only gcp_conn_id parameter. "
+                "Will be used bigquery_conn_id or google_cloud_storage_conn_id if gcp_conn_id not passed.",
+                DeprecationWarning,
+                stacklevel=2,
+            )
 
         super().__init__(**kwargs)
 
@@ -209,8 +220,7 @@ class GCSToBigQueryOperator(BaseOperator):
         self.encoding = encoding
 
         self.max_id_key = max_id_key
-        self.bigquery_conn_id = bigquery_conn_id
-        self.google_cloud_storage_conn_id = google_cloud_storage_conn_id
+        self.gcp_conn_id = gcp_conn_id or bigquery_conn_id or google_cloud_storage_conn_id
         self.delegate_to = delegate_to
 
         self.schema_update_options = schema_update_options
@@ -227,7 +237,7 @@ class GCSToBigQueryOperator(BaseOperator):
 
     def execute(self, context: 'Context'):
         bq_hook = BigQueryHook(
-            bigquery_conn_id=self.bigquery_conn_id,
+            gcp_conn_id=self.gcp_conn_id,
             delegate_to=self.delegate_to,
             location=self.location,
             impersonation_chain=self.impersonation_chain,
@@ -236,7 +246,7 @@ class GCSToBigQueryOperator(BaseOperator):
         if not self.schema_fields:
             if self.schema_object and self.source_format != 'DATASTORE_BACKUP':
                 gcs_hook = GCSHook(
-                    gcp_conn_id=self.google_cloud_storage_conn_id,
+                    gcp_conn_id=self.gcp_conn_id,
                     delegate_to=self.delegate_to,
                     impersonation_chain=self.impersonation_chain,
                 )
@@ -247,7 +257,6 @@ class GCSToBigQueryOperator(BaseOperator):
                 schema_fields = json.loads(blob.decode("utf-8"))
             else:
                 schema_fields = None
-
         else:
             schema_fields = self.schema_fields
 
@@ -255,64 +264,66 @@ class GCSToBigQueryOperator(BaseOperator):
             self.source_objects if isinstance(self.source_objects, list) else [self.source_objects]
         )
         source_uris = [f'gs://{self.bucket}/{source_object}' for source_object in self.source_objects]
-        conn = bq_hook.get_conn()
-        cursor = conn.cursor()
 
         if self.external_table:
-            cursor.create_external_table(
-                external_project_dataset_table=self.destination_project_dataset_table,
-                schema_fields=schema_fields,
-                source_uris=source_uris,
-                source_format=self.source_format,
-                compression=self.compression,
-                skip_leading_rows=self.skip_leading_rows,
-                field_delimiter=self.field_delimiter,
-                max_bad_records=self.max_bad_records,
-                quote_character=self.quote_character,
-                ignore_unknown_values=self.ignore_unknown_values,
-                allow_quoted_newlines=self.allow_quoted_newlines,
-                allow_jagged_rows=self.allow_jagged_rows,
-                encoding=self.encoding,
-                src_fmt_configs=self.src_fmt_configs,
-                encryption_configuration=self.encryption_configuration,
-                labels=self.labels,
-                description=self.description,
-            )
-        else:
-            cursor.run_load(
-                destination_project_dataset_table=self.destination_project_dataset_table,
-                schema_fields=schema_fields,
-                source_uris=source_uris,
-                source_format=self.source_format,
-                autodetect=self.autodetect,
-                create_disposition=self.create_disposition,
-                skip_leading_rows=self.skip_leading_rows,
-                write_disposition=self.write_disposition,
-                field_delimiter=self.field_delimiter,
-                max_bad_records=self.max_bad_records,
-                quote_character=self.quote_character,
-                ignore_unknown_values=self.ignore_unknown_values,
-                allow_quoted_newlines=self.allow_quoted_newlines,
-                allow_jagged_rows=self.allow_jagged_rows,
-                encoding=self.encoding,
-                schema_update_options=self.schema_update_options,
-                src_fmt_configs=self.src_fmt_configs,
-                time_partitioning=self.time_partitioning,
-                cluster_fields=self.cluster_fields,
-                encryption_configuration=self.encryption_configuration,
-                labels=self.labels,
-                description=self.description,
-            )
-
-        if cursor.use_legacy_sql:
-            escaped_table_name = f'[{self.destination_project_dataset_table}]'
+            with warnings.catch_warnings():
+                warnings.simplefilter("ignore", DeprecationWarning)
+                bq_hook.create_external_table(
+                    external_project_dataset_table=self.destination_project_dataset_table,
+                    schema_fields=schema_fields,
+                    source_uris=source_uris,
+                    source_format=self.source_format,
+                    compression=self.compression,
+                    skip_leading_rows=self.skip_leading_rows,
+                    field_delimiter=self.field_delimiter,
+                    max_bad_records=self.max_bad_records,
+                    quote_character=self.quote_character,
+                    ignore_unknown_values=self.ignore_unknown_values,
+                    allow_quoted_newlines=self.allow_quoted_newlines,
+                    allow_jagged_rows=self.allow_jagged_rows,
+                    encoding=self.encoding,
+                    src_fmt_configs=self.src_fmt_configs,
+                    encryption_configuration=self.encryption_configuration,
+                    labels=self.labels,
+                    description=self.description,
+                )
         else:
-            escaped_table_name = f'`{self.destination_project_dataset_table}`'
+            with warnings.catch_warnings():
+                warnings.simplefilter("ignore", DeprecationWarning)
+                bq_hook.run_load(
+                    destination_project_dataset_table=self.destination_project_dataset_table,
+                    schema_fields=schema_fields,
+                    source_uris=source_uris,
+                    source_format=self.source_format,
+                    autodetect=self.autodetect,
+                    create_disposition=self.create_disposition,
+                    skip_leading_rows=self.skip_leading_rows,
+                    write_disposition=self.write_disposition,
+                    field_delimiter=self.field_delimiter,
+                    max_bad_records=self.max_bad_records,
+                    quote_character=self.quote_character,
+                    ignore_unknown_values=self.ignore_unknown_values,
+                    allow_quoted_newlines=self.allow_quoted_newlines,
+                    allow_jagged_rows=self.allow_jagged_rows,
+                    encoding=self.encoding,
+                    schema_update_options=self.schema_update_options,
+                    src_fmt_configs=self.src_fmt_configs,
+                    time_partitioning=self.time_partitioning,
+                    cluster_fields=self.cluster_fields,
+                    encryption_configuration=self.encryption_configuration,
+                    labels=self.labels,
+                    description=self.description,
+                )
 
         if self.max_id_key:
-            select_command = f'SELECT MAX({self.max_id_key}) FROM {escaped_table_name}'
-            cursor.execute(select_command)
-            row = cursor.fetchone()
+            select_command = f'SELECT MAX({self.max_id_key}) FROM `{self.destination_project_dataset_table}`'
+            with warnings.catch_warnings():
+                warnings.simplefilter("ignore", DeprecationWarning)
+                job_id = bq_hook.run_query(
+                    sql=select_command,
+                    use_legacy_sql=False,
+                )
+            row = list(bq_hook.get_job(job_id).result())
             if row:
                 max_id = row[0] if row[0] else 0
                 self.log.info(
@@ -322,4 +333,4 @@ class GCSToBigQueryOperator(BaseOperator):
                     max_id,
                 )
             else:
-                raise RuntimeError(f"The f{select_command} returned no rows!")
+                raise RuntimeError(f"The {select_command} returned no rows!")
diff --git a/tests/providers/google/cloud/transfers/test_bigquery_to_bigquery.py b/tests/providers/google/cloud/transfers/test_bigquery_to_bigquery.py
index b3c0acd..a399546 100644
--- a/tests/providers/google/cloud/transfers/test_bigquery_to_bigquery.py
+++ b/tests/providers/google/cloud/transfers/test_bigquery_to_bigquery.py
@@ -47,7 +47,7 @@ class TestBigQueryToBigQueryOperator(unittest.TestCase):
         )
 
         operator.execute(None)
-        mock_hook.return_value.get_conn.return_value.cursor.return_value.run_copy.assert_called_once_with(
+        mock_hook.return_value.run_copy.assert_called_once_with(
             source_project_dataset_tables=source_project_dataset_tables,
             destination_project_dataset_table=destination_project_dataset_table,
             write_disposition=write_disposition,
diff --git a/tests/providers/google/cloud/transfers/test_gcs_to_bigquery.py b/tests/providers/google/cloud/transfers/test_gcs_to_bigquery.py
index 7ef49ba..4330797 100644
--- a/tests/providers/google/cloud/transfers/test_gcs_to_bigquery.py
+++ b/tests/providers/google/cloud/transfers/test_gcs_to_bigquery.py
@@ -33,25 +33,6 @@ DESCRIPTION = "Test Description"
 
 class TestGCSToBigQueryOperator(unittest.TestCase):
     @mock.patch('airflow.providers.google.cloud.transfers.gcs_to_bigquery.BigQueryHook')
-    def test_execute_explicit_project_legacy(self, bq_hook):
-        operator = GCSToBigQueryOperator(
-            task_id=TASK_ID,
-            bucket=TEST_BUCKET,
-            source_objects=TEST_SOURCE_OBJECTS,
-            destination_project_dataset_table=TEST_EXPLICIT_DEST,
-            max_id_key=MAX_ID_KEY,
-        )
-
-        # using legacy SQL
-        bq_hook.return_value.get_conn.return_value.cursor.return_value.use_legacy_sql = True
-
-        operator.execute(None)
-
-        bq_hook.return_value.get_conn.return_value.cursor.return_value.execute.assert_called_once_with(
-            "SELECT MAX(id) FROM [test-project.dataset.table]"
-        )
-
-    @mock.patch('airflow.providers.google.cloud.transfers.gcs_to_bigquery.BigQueryHook')
     def test_execute_explicit_project(self, bq_hook):
         operator = GCSToBigQueryOperator(
             task_id=TASK_ID,
@@ -61,13 +42,13 @@ class TestGCSToBigQueryOperator(unittest.TestCase):
             max_id_key=MAX_ID_KEY,
         )
 
-        # using non-legacy SQL
-        bq_hook.return_value.get_conn.return_value.cursor.return_value.use_legacy_sql = False
+        bq_hook.return_value.get_job.return_value.result.return_value = ('1',)
 
         operator.execute(None)
 
-        bq_hook.return_value.get_conn.return_value.cursor.return_value.execute.assert_called_once_with(
-            "SELECT MAX(id) FROM `test-project.dataset.table`"
+        bq_hook.return_value.run_query.assert_called_once_with(
+            sql="SELECT MAX(id) FROM `test-project.dataset.table`",
+            use_legacy_sql=False,
         )
 
     @mock.patch('airflow.providers.google.cloud.transfers.gcs_to_bigquery.BigQueryHook')
@@ -83,7 +64,7 @@ class TestGCSToBigQueryOperator(unittest.TestCase):
 
         operator.execute(None)
 
-        bq_hook.return_value.get_conn.return_value.cursor.return_value.run_load.assert_called_once_with(
+        bq_hook.return_value.run_load.assert_called_once_with(
             destination_project_dataset_table=mock.ANY,
             schema_fields=mock.ANY,
             source_uris=mock.ANY,
@@ -121,7 +102,7 @@ class TestGCSToBigQueryOperator(unittest.TestCase):
 
         operator.execute(None)
 
-        bq_hook.return_value.get_conn.return_value.cursor.return_value.run_load.assert_called_once_with(
+        bq_hook.return_value.run_load.assert_called_once_with(
             destination_project_dataset_table=mock.ANY,
             schema_fields=mock.ANY,
             source_uris=mock.ANY,
@@ -160,26 +141,25 @@ class TestGCSToBigQueryOperator(unittest.TestCase):
 
         operator.execute(None)
         # fmt: off
-        bq_hook.return_value.get_conn.return_value.cursor.return_value.create_external_table. \
-            assert_called_once_with(
-                external_project_dataset_table=mock.ANY,
-                schema_fields=mock.ANY,
-                source_uris=mock.ANY,
-                source_format=mock.ANY,
-                compression=mock.ANY,
-                skip_leading_rows=mock.ANY,
-                field_delimiter=mock.ANY,
-                max_bad_records=mock.ANY,
-                quote_character=mock.ANY,
-                ignore_unknown_values=mock.ANY,
-                allow_quoted_newlines=mock.ANY,
-                allow_jagged_rows=mock.ANY,
-                encoding=mock.ANY,
-                src_fmt_configs=mock.ANY,
-                encryption_configuration=mock.ANY,
-                labels=LABELS,
-                description=mock.ANY,
-            )
+        bq_hook.return_value.create_external_table.assert_called_once_with(
+            external_project_dataset_table=mock.ANY,
+            schema_fields=mock.ANY,
+            source_uris=mock.ANY,
+            source_format=mock.ANY,
+            compression=mock.ANY,
+            skip_leading_rows=mock.ANY,
+            field_delimiter=mock.ANY,
+            max_bad_records=mock.ANY,
+            quote_character=mock.ANY,
+            ignore_unknown_values=mock.ANY,
+            allow_quoted_newlines=mock.ANY,
+            allow_jagged_rows=mock.ANY,
+            encoding=mock.ANY,
+            src_fmt_configs=mock.ANY,
+            encryption_configuration=mock.ANY,
+            labels=LABELS,
+            description=mock.ANY,
+        )
         # fmt: on
 
     @mock.patch('airflow.providers.google.cloud.transfers.gcs_to_bigquery.BigQueryHook')
@@ -196,26 +176,25 @@ class TestGCSToBigQueryOperator(unittest.TestCase):
 
         operator.execute(None)
         # fmt: off
-        bq_hook.return_value.get_conn.return_value.cursor.return_value.create_external_table. \
-            assert_called_once_with(
-                external_project_dataset_table=mock.ANY,
-                schema_fields=mock.ANY,
-                source_uris=mock.ANY,
-                source_format=mock.ANY,
-                compression=mock.ANY,
-                skip_leading_rows=mock.ANY,
-                field_delimiter=mock.ANY,
-                max_bad_records=mock.ANY,
-                quote_character=mock.ANY,
-                ignore_unknown_values=mock.ANY,
-                allow_quoted_newlines=mock.ANY,
-                allow_jagged_rows=mock.ANY,
-                encoding=mock.ANY,
-                src_fmt_configs=mock.ANY,
-                encryption_configuration=mock.ANY,
-                labels=mock.ANY,
-                description=DESCRIPTION,
-            )
+        bq_hook.return_value.create_external_table.assert_called_once_with(
+            external_project_dataset_table=mock.ANY,
+            schema_fields=mock.ANY,
+            source_uris=mock.ANY,
+            source_format=mock.ANY,
+            compression=mock.ANY,
+            skip_leading_rows=mock.ANY,
+            field_delimiter=mock.ANY,
+            max_bad_records=mock.ANY,
+            quote_character=mock.ANY,
+            ignore_unknown_values=mock.ANY,
+            allow_quoted_newlines=mock.ANY,
+            allow_jagged_rows=mock.ANY,
+            encoding=mock.ANY,
+            src_fmt_configs=mock.ANY,
+            encryption_configuration=mock.ANY,
+            labels=mock.ANY,
+            description=DESCRIPTION,
+        )
         # fmt: on
 
     @mock.patch('airflow.providers.google.cloud.transfers.gcs_to_bigquery.BigQueryHook')
@@ -229,7 +208,7 @@ class TestGCSToBigQueryOperator(unittest.TestCase):
 
         operator.execute(None)
 
-        bq_hook.return_value.get_conn.return_value.cursor.return_value.run_load.assert_called_once_with(
+        bq_hook.return_value.run_load.assert_called_once_with(
             destination_project_dataset_table=mock.ANY,
             schema_fields=mock.ANY,
             source_uris=[f'gs://{TEST_BUCKET}/{source_object}' for source_object in TEST_SOURCE_OBJECTS],
@@ -265,7 +244,7 @@ class TestGCSToBigQueryOperator(unittest.TestCase):
 
         operator.execute(None)
 
-        bq_hook.return_value.get_conn.return_value.cursor.return_value.run_load.assert_called_once_with(
+        bq_hook.return_value.run_load.assert_called_once_with(
             destination_project_dataset_table=mock.ANY,
             schema_fields=mock.ANY,
             source_uris=[f'gs://{TEST_BUCKET}/{TEST_SOURCE_OBJECTS_AS_STRING}'],