You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by pa...@apache.org on 2020/01/24 00:43:40 UTC
[beam] branch pabloem-patch-1 created (now 29c668d)
This is an automated email from the ASF dual-hosted git repository.
pabloem pushed a change to branch pabloem-patch-1
in repository https://gitbox.apache.org/repos/asf/beam.git.
at 29c668d Indefinite retries to wait for a BQ Load job.
This branch includes the following new commits:
new 29c668d Indefinite retries to wait for a BQ Load job.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[beam] 01/01: Indefinite retries to wait for a BQ Load job.
Posted by pa...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
pabloem pushed a commit to branch pabloem-patch-1
in repository https://gitbox.apache.org/repos/asf/beam.git
commit 29c668d54363f017f59af6600da36aa2ddb4c2da
Author: Pablo <pa...@users.noreply.github.com>
AuthorDate: Thu Jan 23 16:43:32 2020 -0800
Indefinite retries to wait for a BQ Load job.
---
sdks/python/apache_beam/io/gcp/bigquery_file_loads.py | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/sdks/python/apache_beam/io/gcp/bigquery_file_loads.py b/sdks/python/apache_beam/io/gcp/bigquery_file_loads.py
index fdce60b..0b6ad95 100644
--- a/sdks/python/apache_beam/io/gcp/bigquery_file_loads.py
+++ b/sdks/python/apache_beam/io/gcp/bigquery_file_loads.py
@@ -504,7 +504,11 @@ class WaitForBQJobs(beam.DoFn):
def process(self, element, dest_ids_list):
job_references = [elm[1] for elm in dest_ids_list]
for ref in job_references:
- self.bq_wrapper.wait_for_bq_job(ref, sleep_duration_sec=10)
+ # We must poll repeatedly until the job finishes or fails, thus setting
+ # max_retries to 0.
+ self.bq_wrapper.wait_for_bq_job(ref,
+ sleep_duration_sec=10,
+ max_retries=0)
return dest_ids_list # Pass the list of destination-jobs downstream