[GitHub] kaxil closed pull request #3901: [AIRFLOW-2772] BigQuery hook allow specifying both the partition fiel…

[GitBox <gi...@apache.org>]

kaxil closed pull request #3901: [AIRFLOW-2772] BigQuery hook allow specifying both the partition fiel…
URL: https://github.com/apache/incubator-airflow/pull/3901
 
 
   

This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:

As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):

diff --git a/airflow/contrib/hooks/bigquery_hook.py b/airflow/contrib/hooks/bigquery_hook.py
index 7a1631b53a..693270fad4 100644
--- a/airflow/contrib/hooks/bigquery_hook.py
+++ b/airflow/contrib/hooks/bigquery_hook.py
@@ -562,9 +562,7 @@ def run_query(self,
             The default value is INTERACTIVE.
         :type priority: str
         :param time_partitioning: configure optional time partitioning fields i.e.
-            partition by field, type and
-            expiration as per API specifications. Note that 'field' is not available in
-            conjunction with dataset.table$partition.
+            partition by field, type and expiration as per API specifications.
         :type time_partitioning: dict
         :param cluster_fields: Request that the result of this query be stored sorted
             by one or more columns. This is only available in combination with
@@ -926,9 +924,7 @@ def run_load(self,
         :param src_fmt_configs: configure optional fields specific to the source format
         :type src_fmt_configs: dict
         :param time_partitioning: configure optional time partitioning fields i.e.
-            partition by field, type and
-            expiration as per API specifications. Note that 'field' is not available in
-            conjunction with dataset.table$partition.
+            partition by field, type and  expiration as per API specifications.
         :type time_partitioning: dict
         :param cluster_fields: Request that the result of this load be stored sorted
             by one or more columns. This is only available in combination with
@@ -1662,12 +1658,7 @@ def _cleanse_time_partitioning(destination_dataset_table, time_partitioning_in):
 
     time_partitioning_out = {}
     if destination_dataset_table and '$' in destination_dataset_table:
-        if time_partitioning_in.get('field'):
-            raise ValueError(
-                "Cannot specify field partition and partition name"
-                "(dataset.table$partition) at the same time")
         time_partitioning_out['type'] = 'DAY'
-
     time_partitioning_out.update(time_partitioning_in)
     return time_partitioning_out
 
diff --git a/airflow/contrib/operators/bigquery_operator.py b/airflow/contrib/operators/bigquery_operator.py
index fec877db05..8eee6a395c 100644
--- a/airflow/contrib/operators/bigquery_operator.py
+++ b/airflow/contrib/operators/bigquery_operator.py
@@ -96,9 +96,7 @@ class BigQueryOperator(BaseOperator):
         The default value is INTERACTIVE.
     :type priority: str
     :param time_partitioning: configure optional time partitioning fields i.e.
-        partition by field, type and
-        expiration as per API specifications. Note that 'field' is not available in
-        conjunction with dataset.table$partition.
+        partition by field, type and expiration as per API specifications.
     :type time_partitioning: dict
     :param cluster_fields: Request that the result of this query be stored sorted
         by one or more columns. This is only available in conjunction with
diff --git a/tests/contrib/hooks/test_bigquery_hook.py b/tests/contrib/hooks/test_bigquery_hook.py
index e1379dde79..0006b0c616 100644
--- a/tests/contrib/hooks/test_bigquery_hook.py
+++ b/tests/contrib/hooks/test_bigquery_hook.py
@@ -447,13 +447,6 @@ def test_extra_time_partitioning_options(self):
         }
         self.assertEqual(tp_out, expect)
 
-    def test_cant_add_dollar_and_field_name(self):
-        with self.assertRaises(ValueError):
-            _cleanse_time_partitioning(
-                'test.teast$20170101',
-                {'type': 'DAY', 'field': 'test_field', 'expirationMs': 1000}
-            )
-
 
 class TestClusteringInRunJob(unittest.TestCase):
 


 

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services