[beam] branch master updated: [BEAM-5915] Timeout http requests, if they got stuck, at dataflow job creation time (#6922)

[al...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

altay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
     new 41bf693  [BEAM-5915] Timeout http requests, if they got stuck, at dataflow job creation time (#6922)
41bf693 is described below

commit 41bf69391fe2db032302960d2188134563d6653d
Author: Ahmet Altay <aa...@gmail.com>
AuthorDate: Tue Nov 6 15:53:33 2018 -0800

    [BEAM-5915] Timeout http requests, if they got stuck, at dataflow job creation time (#6922)
    
    * Timeout http requests, if they got stuck, at dataflow job creation time
---
 sdks/python/apache_beam/runners/dataflow/internal/apiclient.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py b/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py
index fbd2d41..b833aaa 100644
--- a/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py
+++ b/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py
@@ -24,6 +24,7 @@ from __future__ import absolute_import
 from builtins import object
 import codecs
 import getpass
+import httplib2
 import json
 import logging
 import os
@@ -425,14 +426,19 @@ class DataflowApplicationClient(object):
       credentials = None
     else:
       credentials = get_service_credentials()
+
+    # Use 60 second socket timeout avoid hangs during network flakiness.
+    http_client = httplib2.Http(timeout=60)
     self._client = dataflow.DataflowV1b3(
         url=self.google_cloud_options.dataflow_endpoint,
         credentials=credentials,
-        get_credentials=(not self.google_cloud_options.no_auth))
+        get_credentials=(not self.google_cloud_options.no_auth),
+        http=http_client)
     self._storage_client = storage.StorageV1(
         url='https://www.googleapis.com/storage/v1',
         credentials=credentials,
-        get_credentials=(not self.google_cloud_options.no_auth))
+        get_credentials=(not self.google_cloud_options.no_auth),
+        http=http_client)
 
   # TODO(silviuc): Refactor so that retry logic can be applied.
   @retry.no_retries  # Using no_retries marks this as an integration point.