You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by cr...@apache.org on 2017/10/04 05:30:10 UTC
[2/2] incubator-airflow git commit: [AIRFLOW-1635] Allow creating GCP
connection without requiring a JSON file
[AIRFLOW-1635] Allow creating GCP connection without requiring a JSON file
Closes #2640 from barrywhart/airflow-1635-gcp-
json-data-master
Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/6dec7acd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/6dec7acd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/6dec7acd
Branch: refs/heads/v1-9-test
Commit: 6dec7acde3b599599cd83cfad3c84fc014b9931e
Parents: 50e31f0
Author: Barry Hart <ba...@yahoo.com>
Authored: Thu Sep 28 09:19:43 2017 -0700
Committer: Chris Riccomini <cr...@apache.org>
Committed: Tue Oct 3 22:30:03 2017 -0700
----------------------------------------------------------------------
airflow/contrib/hooks/gcp_api_base_hook.py | 27 +++++++++++++++++++++++--
airflow/www/views.py | 2 ++
2 files changed, 27 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/6dec7acd/airflow/contrib/hooks/gcp_api_base_hook.py
----------------------------------------------------------------------
diff --git a/airflow/contrib/hooks/gcp_api_base_hook.py b/airflow/contrib/hooks/gcp_api_base_hook.py
index 28721d3..e6ca240 100644
--- a/airflow/contrib/hooks/gcp_api_base_hook.py
+++ b/airflow/contrib/hooks/gcp_api_base_hook.py
@@ -12,6 +12,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
+import json
+
import httplib2
from oauth2client.client import GoogleCredentials
from oauth2client.service_account import ServiceAccountCredentials
@@ -59,20 +61,23 @@ class GoogleCloudBaseHook(BaseHook, LoggingMixin):
Returns the Credentials object for Google API
"""
key_path = self._get_field('key_path', False)
+ keyfile_dict = self._get_field('keyfile_dict', False)
scope = self._get_field('scope', False)
kwargs = {}
if self.delegate_to:
kwargs['sub'] = self.delegate_to
- if not key_path:
+ if not key_path and not keyfile_dict:
self.log.info('Getting connection using `gcloud auth` user, since no key file '
'is defined for hook.')
credentials = GoogleCredentials.get_application_default()
- else:
+ elif key_path:
if not scope:
raise AirflowException('Scope should be defined when using a key file.')
scopes = [s.strip() for s in scope.split(',')]
+
+ # Get credentials from a JSON file.
if key_path.endswith('.json'):
self.log.info('Getting connection using a JSON key file.')
credentials = ServiceAccountCredentials\
@@ -82,6 +87,24 @@ class GoogleCloudBaseHook(BaseHook, LoggingMixin):
'use a JSON key file.')
else:
raise AirflowException('Unrecognised extension for key file.')
+ else:
+ if not scope:
+ raise AirflowException('Scope should be defined when using key JSON.')
+ scopes = [s.strip() for s in scope.split(',')]
+
+ # Get credentials from JSON data provided in the UI.
+ try:
+ keyfile_dict = json.loads(keyfile_dict)
+
+ # Depending on how the JSON was formatted, it may contain
+ # escaped newlines. Convert those to actual newlines.
+ keyfile_dict['private_key'] = keyfile_dict['private_key'].replace(
+ '\\n', '\n')
+
+ credentials = ServiceAccountCredentials\
+ .from_json_keyfile_dict(keyfile_dict, scopes)
+ except json.decoder.JSONDecodeError:
+ raise AirflowException('Invalid key JSON.')
return credentials
def _get_access_token(self):
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/6dec7acd/airflow/www/views.py
----------------------------------------------------------------------
diff --git a/airflow/www/views.py b/airflow/www/views.py
index a0a0999..ad27238 100644
--- a/airflow/www/views.py
+++ b/airflow/www/views.py
@@ -2582,6 +2582,7 @@ class ConnectionModelView(wwwutils.SuperUserMixin, AirflowModelView):
'extra__jdbc__drv_clsname',
'extra__google_cloud_platform__project',
'extra__google_cloud_platform__key_path',
+ 'extra__google_cloud_platform__keyfile_dict',
'extra__google_cloud_platform__scope',
)
verbose_name = "Connection"
@@ -2603,6 +2604,7 @@ class ConnectionModelView(wwwutils.SuperUserMixin, AirflowModelView):
'extra__jdbc__drv_clsname': StringField('Driver Class'),
'extra__google_cloud_platform__project': StringField('Project Id'),
'extra__google_cloud_platform__key_path': StringField('Keyfile Path'),
+ 'extra__google_cloud_platform__keyfile_dict': PasswordField('Keyfile JSON'),
'extra__google_cloud_platform__scope': StringField('Scopes (comma seperated)'),
}