You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by dh...@apache.org on 2016/09/06 18:31:47 UTC
[2/2] incubator-beam git commit: BigQuery: limit max job polling time
to 1 minute
BigQuery: limit max job polling time to 1 minute
Before the backoff would grow unboundedly, so we could in principle wait
1.5x to 2x the actual job time. For long running jobs this is hours.
Now, we just back off at most 1 minute between checking the job state.
Note there should be no danger of QPS overload here because we should
have very few concurrent outstanding jobs
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/8d306bbf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/8d306bbf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/8d306bbf
Branch: refs/heads/master
Commit: 8d306bbf5481ff1a5d27a2ae8d73e710532154a8
Parents: be689df
Author: Dan Halperin <dh...@google.com>
Authored: Sun Sep 4 14:54:42 2016 -0700
Committer: Dan Halperin <dh...@google.com>
Committed: Tue Sep 6 11:31:38 2016 -0700
----------------------------------------------------------------------
.../apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImpl.java | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/8d306bbf/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImpl.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImpl.java
index 8b5e8c2..20dadff 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImpl.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImpl.java
@@ -232,11 +232,12 @@ class BigQueryServicesImpl implements BigQueryServices {
}
@Override
- public Job pollJob(JobReference jobRef, int maxAttempts)
- throws InterruptedException {
+ public Job pollJob(JobReference jobRef, int maxAttempts) throws InterruptedException {
BackOff backoff =
FluentBackoff.DEFAULT
- .withMaxRetries(maxAttempts).withInitialBackoff(INITIAL_JOB_STATUS_POLL_BACKOFF)
+ .withMaxRetries(maxAttempts)
+ .withInitialBackoff(INITIAL_JOB_STATUS_POLL_BACKOFF)
+ .withMaxBackoff(Duration.standardMinutes(1))
.backoff();
return pollJob(jobRef, Sleeper.DEFAULT, backoff);
}