You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by cr...@apache.org on 2017/09/26 23:07:36 UTC
incubator-airflow git commit: [AIRFLOW-1635] Allow creating GCP
connection without requiring a JSON file
Repository: incubator-airflow
Updated Branches:
refs/heads/v1-8-test 6069075a7 -> 50d0d5b0b
[AIRFLOW-1635] Allow creating GCP connection without requiring a JSON file
Closes #2634 from barrywhart/airflow-1635-gcp-
json-data
Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/50d0d5b0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/50d0d5b0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/50d0d5b0
Branch: refs/heads/v1-8-test
Commit: 50d0d5b0bcbbd18d2c76d788e22bfdcc2c0d6f71
Parents: 6069075
Author: Barry Hart <ba...@yahoo.com>
Authored: Tue Sep 26 16:07:21 2017 -0700
Committer: Chris Riccomini <cr...@apache.org>
Committed: Tue Sep 26 16:07:21 2017 -0700
----------------------------------------------------------------------
airflow/contrib/hooks/gcp_api_base_hook.py | 26 +++++++++++++++++++++++--
airflow/www/views.py | 2 ++
2 files changed, 26 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/50d0d5b0/airflow/contrib/hooks/gcp_api_base_hook.py
----------------------------------------------------------------------
diff --git a/airflow/contrib/hooks/gcp_api_base_hook.py b/airflow/contrib/hooks/gcp_api_base_hook.py
index 2260e7b..dbe076f 100644
--- a/airflow/contrib/hooks/gcp_api_base_hook.py
+++ b/airflow/contrib/hooks/gcp_api_base_hook.py
@@ -13,6 +13,7 @@
# limitations under the License.
#
+import json
import logging
import httplib2
@@ -63,20 +64,23 @@ class GoogleCloudBaseHook(BaseHook):
service hook connection.
"""
key_path = self._get_field('key_path', False)
+ keyfile_dict = self._get_field('keyfile_dict', False)
scope = self._get_field('scope', False)
kwargs = {}
if self.delegate_to:
kwargs['sub'] = self.delegate_to
- if not key_path:
+ if not key_path and not keyfile_dict:
logging.info('Getting connection using `gcloud auth` user, since no key file '
'is defined for hook.')
credentials = GoogleCredentials.get_application_default()
- else:
+ elif key_path:
if not scope:
raise AirflowException('Scope should be defined when using a key file.')
scopes = [s.strip() for s in scope.split(',')]
+
+ # Get credentials from a JSON file.
if key_path.endswith('.json'):
logging.info('Getting connection using a JSON key file.')
credentials = ServiceAccountCredentials\
@@ -86,6 +90,24 @@ class GoogleCloudBaseHook(BaseHook):
'use a JSON key file.')
else:
raise AirflowException('Unrecognised extension for key file.')
+ else:
+ if not scope:
+ raise AirflowException('Scope should be defined when using key JSON.')
+ scopes = [s.strip() for s in scope.split(',')]
+
+ # Get credentials from JSON data provided in the UI.
+ try:
+ keyfile_dict = json.loads(keyfile_dict)
+
+ # Depending on how the JSON was formatted, it may contain
+ # escaped newlines. Convert those to actual newlines.
+ keyfile_dict['private_key'] = keyfile_dict['private_key'].replace(
+ '\\n', '\n')
+
+ credentials = ServiceAccountCredentials\
+ .from_json_keyfile_dict(keyfile_dict, scopes)
+ except json.decoder.JSONDecodeError:
+ raise AirflowException('Invalid key JSON.')
http = httplib2.Http()
return credentials.authorize(http)
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/50d0d5b0/airflow/www/views.py
----------------------------------------------------------------------
diff --git a/airflow/www/views.py b/airflow/www/views.py
index a873e63..f55e310 100644
--- a/airflow/www/views.py
+++ b/airflow/www/views.py
@@ -2469,6 +2469,7 @@ class ConnectionModelView(wwwutils.SuperUserMixin, AirflowModelView):
'extra__jdbc__drv_clsname',
'extra__google_cloud_platform__project',
'extra__google_cloud_platform__key_path',
+ 'extra__google_cloud_platform__keyfile_dict',
'extra__google_cloud_platform__scope',
)
verbose_name = "Connection"
@@ -2490,6 +2491,7 @@ class ConnectionModelView(wwwutils.SuperUserMixin, AirflowModelView):
'extra__jdbc__drv_clsname': StringField('Driver Class'),
'extra__google_cloud_platform__project': StringField('Project Id'),
'extra__google_cloud_platform__key_path': StringField('Keyfile Path'),
+ 'extra__google_cloud_platform__keyfile_dict': PasswordField('Keyfile JSON'),
'extra__google_cloud_platform__scope': StringField('Scopes (comma seperated)'),
}