[beam] 01/01: Indefinite retries to wait for a BQ Load job.

[pa...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

pabloem pushed a commit to branch pabloem-patch-1
in repository https://gitbox.apache.org/repos/asf/beam.git

commit 29c668d54363f017f59af6600da36aa2ddb4c2da
Author: Pablo <pa...@users.noreply.github.com>
AuthorDate: Thu Jan 23 16:43:32 2020 -0800

    Indefinite retries to wait for a BQ Load job.
---
 sdks/python/apache_beam/io/gcp/bigquery_file_loads.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/sdks/python/apache_beam/io/gcp/bigquery_file_loads.py b/sdks/python/apache_beam/io/gcp/bigquery_file_loads.py
index fdce60b..0b6ad95 100644
--- a/sdks/python/apache_beam/io/gcp/bigquery_file_loads.py
+++ b/sdks/python/apache_beam/io/gcp/bigquery_file_loads.py
@@ -504,7 +504,11 @@ class WaitForBQJobs(beam.DoFn):
   def process(self, element, dest_ids_list):
     job_references = [elm[1] for elm in dest_ids_list]
     for ref in job_references:
-      self.bq_wrapper.wait_for_bq_job(ref, sleep_duration_sec=10)
+      # We must poll repeatedly until the job finishes or fails, thus setting
+      # max_retries to 0.
+      self.bq_wrapper.wait_for_bq_job(ref, 
+                                      sleep_duration_sec=10,
+                                      max_retries=0)
 
     return dest_ids_list  # Pass the list of destination-jobs downstream