You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by bo...@apache.org on 2017/03/28 23:45:27 UTC
incubator-airflow git commit: [AIRFLOW-1034] Make it possible to
connect to S3 in sigv4 regions
Repository: incubator-airflow
Updated Branches:
refs/heads/master e739a5f52 -> 4c0905068
[AIRFLOW-1034] Make it possible to connect to S3 in sigv4 regions
Closes #2181 from buyology/fix-s3-in-sigv4-regions
Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/4c090506
Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/4c090506
Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/4c090506
Branch: refs/heads/master
Commit: 4c0905068d2cdf18b9420013dc502a276a5083ea
Parents: e739a5f
Author: Robin <ro...@formulate.se>
Authored: Tue Mar 28 16:45:02 2017 -0700
Committer: Bolke de Bruin <bo...@xs4all.nl>
Committed: Tue Mar 28 16:45:02 2017 -0700
----------------------------------------------------------------------
airflow/hooks/S3_hook.py | 10 +++++++++-
1 file changed, 9 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/4c090506/airflow/hooks/S3_hook.py
----------------------------------------------------------------------
diff --git a/airflow/hooks/S3_hook.py b/airflow/hooks/S3_hook.py
index 73b1154..caaa575 100644
--- a/airflow/hooks/S3_hook.py
+++ b/airflow/hooks/S3_hook.py
@@ -25,7 +25,7 @@ from urllib.parse import urlparse
import warnings
import boto
-from boto.s3.connection import S3Connection
+from boto.s3.connection import S3Connection, NoHostProvided
from boto.sts import STSConnection
boto.set_stream_logger('boto')
logging.getLogger("boto").setLevel(logging.INFO)
@@ -101,9 +101,12 @@ class S3Hook(BaseHook):
self.extra_params = self.s3_conn.extra_dejson
self.profile = self.extra_params.get('profile')
self.calling_format = None
+ self.s3_host = None
self._creds_in_conn = 'aws_secret_access_key' in self.extra_params
self._creds_in_config_file = 's3_config_file' in self.extra_params
self._default_to_boto = False
+ if 'host' in self.extra_params:
+ self.s3_host = self.extra_params['host']
if self._creds_in_conn:
self._a_key = self.extra_params['aws_access_key_id']
self._s_key = self.extra_params['aws_secret_access_key']
@@ -167,10 +170,14 @@ class S3Hook(BaseHook):
a_key = self._a_key
s_key = self._s_key
calling_format = self.calling_format
+ s3_host = self.s3_host
if calling_format is None:
calling_format = 'boto.s3.connection.SubdomainCallingFormat'
+ if s3_host is None:
+ s3_host = NoHostProvided
+
if self._sts_conn_required:
sts_connection = STSConnection(aws_access_key_id=a_key,
aws_secret_access_key=s_key,
@@ -190,6 +197,7 @@ class S3Hook(BaseHook):
connection = S3Connection(aws_access_key_id=a_key,
aws_secret_access_key=s_key,
calling_format=calling_format,
+ host=s3_host,
profile_name=self.profile)
return connection