You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by GitBox <gi...@apache.org> on 2021/03/14 13:46:45 UTC
[GitHub] [airflow] sschaetz commented on a change in pull request #13996: Add GCS timespan transform operator
sschaetz commented on a change in pull request #13996:
URL: https://github.com/apache/airflow/pull/13996#discussion_r593903447
##########
File path: airflow/providers/google/cloud/hooks/gcs.py
##########
@@ -285,20 +290,43 @@ def download(
:type chunk_size: int
:param timeout: Request timeout in seconds.
:type timeout: int
+ :param num_max_attempts: Number of attempts to download the file.
+ :type num_max_attempts: int
"""
# TODO: future improvement check file size before downloading,
# to check for local space availability
- client = self.get_conn()
- bucket = client.bucket(bucket_name)
- blob = bucket.blob(blob_name=object_name, chunk_size=chunk_size)
-
- if filename:
- blob.download_to_filename(filename, timeout=timeout)
- self.log.info('File downloaded to %s', filename)
- return filename
- else:
- return blob.download_as_string()
+ num_file_attempts = 0
+
+ while num_file_attempts < num_max_attempts:
+ try:
+ num_file_attempts += 1
+ client = self.get_conn()
+ bucket = client.bucket(bucket_name)
+ blob = bucket.blob(blob_name=object_name, chunk_size=chunk_size)
+
+ if filename:
+ blob.download_to_filename(filename, timeout=timeout)
+ self.log.info('File downloaded to %s', filename)
+ return filename
+ else:
+ return blob.download_as_string()
+
+ except GoogleCloudError as e:
Review comment:
Thanks - fixed now!
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org