You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by ma...@apache.org on 2020/09/22 21:07:33 UTC
[airavata-django-portal] branch
AIRAVATA-3346-implement-remote-fs-abstraction-of-user-storage updated
(5c9ba70 -> c592887)
This is an automated email from the ASF dual-hosted git repository.
machristie pushed a change to branch AIRAVATA-3346-implement-remote-fs-abstraction-of-user-storage
in repository https://gitbox.apache.org/repos/asf/airavata-django-portal.git.
from 5c9ba70 AIRAVATA-3346 Add missing allowed http methods
new 88a3e9b AIRAVATA-3346 Use remote API to launch experiment and get relative data directory
new bdb39af AIRAVATA-3346 Proxy clone request when use remote data store since input files must be copied on the remote portal instance
new 62b47f6 AIRAVATA-3346 Removing no longer used, deprecated user_file_exists
new a0f6733 AIRAVATA-3346 Fixing missing function
new c592887 AIRAVATA-3346 Handle failed bearer token authentication
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
django_airavata/apps/api/authentication.py | 3 +
django_airavata/apps/api/serializers.py | 61 ++++++++++++----
django_airavata/apps/api/views.py | 110 +++++++++++++++++++----------
django_airavata/apps/auth/backends.py | 6 ++
django_airavata/apps/auth/views.py | 4 ++
django_airavata/apps/workspace/views.py | 6 --
6 files changed, 135 insertions(+), 55 deletions(-)
[airavata-django-portal] 03/05: AIRAVATA-3346 Removing no longer
used, deprecated user_file_exists
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
machristie pushed a commit to branch AIRAVATA-3346-implement-remote-fs-abstraction-of-user-storage
in repository https://gitbox.apache.org/repos/asf/airavata-django-portal.git
commit 62b47f673b6564fa2fe4eb804db24ee75c1f3723
Author: Marcus Christie <ma...@apache.org>
AuthorDate: Wed Sep 16 14:44:51 2020 -0400
AIRAVATA-3346 Removing no longer used, deprecated user_file_exists
---
django_airavata/apps/workspace/views.py | 6 ------
1 file changed, 6 deletions(-)
diff --git a/django_airavata/apps/workspace/views.py b/django_airavata/apps/workspace/views.py
index b358c3c..2d5a371 100644
--- a/django_airavata/apps/workspace/views.py
+++ b/django_airavata/apps/workspace/views.py
@@ -99,12 +99,6 @@ def create_experiment(request, app_module_id):
except Exception as e:
logger.exception(
f"Failed checking data product uri: {dp_uri}")
- else:
- # TODO: remove this functionality, data product URI should be passed instead
- data_product_uri = user_storage_sdk.user_file_exists(
- request, user_file_url.path)
- if data_product_uri is not None:
- user_input_values[app_input['name']] = data_product_uri
except ValueError as e:
logger.exception(f"Invalid user file value: {user_file_value}")
elif (app_input['type'] == DataType.STRING and
[airavata-django-portal] 04/05: AIRAVATA-3346 Fixing missing
function
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
machristie pushed a commit to branch AIRAVATA-3346-implement-remote-fs-abstraction-of-user-storage
in repository https://gitbox.apache.org/repos/asf/airavata-django-portal.git
commit a0f6733279387e03ffef738bcb2f718af74bc3d4
Author: Marcus Christie <ma...@apache.org>
AuthorDate: Wed Sep 16 14:51:04 2020 -0400
AIRAVATA-3346 Fixing missing function
---
django_airavata/apps/api/views.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/django_airavata/apps/api/views.py b/django_airavata/apps/api/views.py
index c2d5c08..404117b 100644
--- a/django_airavata/apps/api/views.py
+++ b/django_airavata/apps/api/views.py
@@ -1565,7 +1565,7 @@ class UserStoragePathView(APIView):
# Replace the file if the request has a file upload.
if 'file' in request.FILES:
self.delete(request=request, path=path, format=format)
- dir_path, file_name = split_dir_path_and_file_name(path=path)
+ dir_path, file_name = os.path.split(path)
self.post(request=request, path=dir_path, format=format, file_name=file_name)
# Replace only the file content if the request body has the `fileContentText`
elif request.data and "fileContentText" in request.data:
[airavata-django-portal] 02/05: AIRAVATA-3346 Proxy clone request
when use remote data store since input files must be copied on the remote
portal instance
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
machristie pushed a commit to branch AIRAVATA-3346-implement-remote-fs-abstraction-of-user-storage
in repository https://gitbox.apache.org/repos/asf/airavata-django-portal.git
commit bdb39af7a8595b04a2f267d2dae4f028af516913
Author: Marcus Christie <ma...@apache.org>
AuthorDate: Wed Sep 16 14:34:26 2020 -0400
AIRAVATA-3346 Proxy clone request when use remote data store since input files must be copied on the remote portal instance
---
django_airavata/apps/api/views.py | 65 ++++++++++++++++++++++++---------------
1 file changed, 40 insertions(+), 25 deletions(-)
diff --git a/django_airavata/apps/api/views.py b/django_airavata/apps/api/views.py
index 90273bb..c2d5c08 100644
--- a/django_airavata/apps/api/views.py
+++ b/django_airavata/apps/api/views.py
@@ -323,31 +323,46 @@ class ExperimentViewSet(APIBackedViewSet):
@detail_route(methods=['post'])
def clone(self, request, experiment_id=None):
-
- # figure what project to clone into
- experiment = self.request.airavata_client.getExperiment(
- self.authz_token, experiment_id)
- project_id = self._get_writeable_project(experiment)
-
- # clone experiment
- cloned_experiment_id = request.airavata_client.cloneExperiment(
- self.authz_token, experiment_id,
- "Clone of {}".format(experiment.experimentName), project_id)
- cloned_experiment = request.airavata_client.getExperiment(
- self.authz_token, cloned_experiment_id)
-
- # Create a copy of the experiment input files
- self._copy_cloned_experiment_input_uris(cloned_experiment)
-
- # Null out experimentDataDir so a new one will get created at launch
- # time
- cloned_experiment.userConfigurationData.experimentDataDir = None
- request.airavata_client.updateExperiment(
- self.authz_token, cloned_experiment.experimentId, cloned_experiment
- )
- serializer = self.serializer_class(
- cloned_experiment, context={'request': request})
- return Response(serializer.data)
+ if getattr(
+ settings,
+ 'GATEWAY_DATA_STORE_REMOTE_API',
+ None) is not None:
+ # Proxy the clone/ request to the remote Django portal instance
+ # since it must locally copy input files, which are only on the
+ # remote Django portal instance
+ headers = {
+ 'Authorization': f'Bearer {request.authz_token.accessToken}'}
+ r = requests.post(
+ f'{settings.GATEWAY_DATA_STORE_REMOTE_API}/experiments/{quote(experiment_id)}/clone/',
+ headers=headers,
+ )
+ r.raise_for_status()
+ return Response(r.json())
+ else:
+ # figure what project to clone into
+ experiment = self.request.airavata_client.getExperiment(
+ self.authz_token, experiment_id)
+ project_id = self._get_writeable_project(experiment)
+
+ # clone experiment
+ cloned_experiment_id = request.airavata_client.cloneExperiment(
+ self.authz_token, experiment_id,
+ "Clone of {}".format(experiment.experimentName), project_id)
+ cloned_experiment = request.airavata_client.getExperiment(
+ self.authz_token, cloned_experiment_id)
+
+ # Create a copy of the experiment input files
+ self._copy_cloned_experiment_input_uris(cloned_experiment)
+
+ # Null out experimentDataDir so a new one will get created at launch
+ # time
+ cloned_experiment.userConfigurationData.experimentDataDir = None
+ request.airavata_client.updateExperiment(
+ self.authz_token, cloned_experiment.experimentId, cloned_experiment
+ )
+ serializer = self.serializer_class(
+ cloned_experiment, context={'request': request})
+ return Response(serializer.data)
@detail_route(methods=['post'])
def cancel(self, request, experiment_id=None):
[airavata-django-portal] 01/05: AIRAVATA-3346 Use remote API to
launch experiment and get relative data directory
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
machristie pushed a commit to branch AIRAVATA-3346-implement-remote-fs-abstraction-of-user-storage
in repository https://gitbox.apache.org/repos/asf/airavata-django-portal.git
commit 88a3e9b86055d01df484979d190c2cded3c0dff1
Author: Marcus Christie <ma...@apache.org>
AuthorDate: Wed Sep 16 14:22:27 2020 -0400
AIRAVATA-3346 Use remote API to launch experiment and get relative data directory
---
django_airavata/apps/api/serializers.py | 61 ++++++++++++++++++++++++++-------
django_airavata/apps/api/views.py | 45 ++++++++++++++++++------
django_airavata/apps/auth/views.py | 4 +++
3 files changed, 86 insertions(+), 24 deletions(-)
diff --git a/django_airavata/apps/api/serializers.py b/django_airavata/apps/api/serializers.py
index 5622c65..0ecd926 100644
--- a/django_airavata/apps/api/serializers.py
+++ b/django_airavata/apps/api/serializers.py
@@ -64,6 +64,7 @@ from airavata.model.workspace.ttypes import (
from airavata_django_portal_sdk import user_storage
from . import models, thrift_utils
+import requests
log = logging.getLogger(__name__)
@@ -457,11 +458,26 @@ class ExperimentSerializer(
ResourcePermissionType.WRITE)
def get_relativeExperimentDataDir(self, experiment):
+
if (experiment.userConfigurationData and
experiment.userConfigurationData.experimentDataDir):
request = self.context['request']
data_dir = experiment.userConfigurationData.experimentDataDir
- if user_storage.dir_exists(request, data_dir):
+ if getattr(
+ settings,
+ 'GATEWAY_DATA_STORE_REMOTE_API',
+ None) is not None:
+ # Load the relativeExperimentDataDir from the remote Django
+ # portal instance
+ headers = {
+ 'Authorization': f'Bearer {request.authz_token.accessToken}'}
+ r = requests.get(
+ f'{settings.GATEWAY_DATA_STORE_REMOTE_API}/experiments/{quote(experiment.experimentId)}/',
+ headers=headers,
+ )
+ r.raise_for_status()
+ return r.json()['relativeExperimentDataDir']
+ elif user_storage.dir_exists(request, data_dir):
return user_storage.get_rel_path(request, data_dir)
else:
return None
@@ -673,7 +689,8 @@ class SharedEntitySerializer(serializers.Serializer):
raise Exception("Not implemented")
def update(self, instance, validated_data):
- # Compute lists of ids to grant/revoke READ/WRITE/MANAGE_SHARING permission
+ # Compute lists of ids to grant/revoke READ/WRITE/MANAGE_SHARING
+ # permission
existing_user_permissions = {
user['user'].airavataInternalUserId: user['permissionType']
for user in instance['userPermissions']}
@@ -682,10 +699,15 @@ class SharedEntitySerializer(serializers.Serializer):
user['permissionType']
for user in validated_data['userPermissions']}
- (user_grant_read_permission, user_grant_write_permission, user_grant_manage_sharing_permission,
- user_revoke_read_permission, user_revoke_write_permission, user_revoke_manage_sharing_permission) = \
- self._compute_all_revokes_and_grants(existing_user_permissions,
- new_user_permissions)
+ (
+ user_grant_read_permission,
+ user_grant_write_permission,
+ user_grant_manage_sharing_permission,
+ user_revoke_read_permission,
+ user_revoke_write_permission,
+ user_revoke_manage_sharing_permission) = self._compute_all_revokes_and_grants(
+ existing_user_permissions,
+ new_user_permissions)
existing_group_permissions = {
group['group'].id: group['permissionType']
@@ -694,10 +716,15 @@ class SharedEntitySerializer(serializers.Serializer):
group['group']['id']: group['permissionType']
for group in validated_data['groupPermissions']}
- (group_grant_read_permission, group_grant_write_permission, group_grant_manage_sharing_permission,
- group_revoke_read_permission, group_revoke_write_permission, group_revoke_manage_sharing_permission) = \
- self._compute_all_revokes_and_grants(existing_group_permissions,
- new_group_permissions)
+ (
+ group_grant_read_permission,
+ group_grant_write_permission,
+ group_grant_manage_sharing_permission,
+ group_revoke_read_permission,
+ group_revoke_write_permission,
+ group_revoke_manage_sharing_permission) = self._compute_all_revokes_and_grants(
+ existing_group_permissions,
+ new_group_permissions)
instance['_user_grant_read_permission'] = user_grant_read_permission
instance['_user_grant_write_permission'] = user_grant_write_permission
@@ -749,15 +776,23 @@ class SharedEntitySerializer(serializers.Serializer):
grant_write_permission.append(id)
if ResourcePermissionType.MANAGE_SHARING in grants:
grant_manage_sharing_permission.append(id)
- return (grant_read_permission, grant_write_permission, grant_manage_sharing_permission,
- revoke_read_permission, revoke_write_permission, revoke_manage_sharing_permission)
+ return (
+ grant_read_permission,
+ grant_write_permission,
+ grant_manage_sharing_permission,
+ revoke_read_permission,
+ revoke_write_permission,
+ revoke_manage_sharing_permission)
def _compute_revokes_and_grants(self, current_permission=None,
new_permission=None):
read_permissions = set((ResourcePermissionType.READ,))
write_permissions = set((ResourcePermissionType.READ,
ResourcePermissionType.WRITE))
- manage_share_permissions = set((ResourcePermissionType.READ, ResourcePermissionType.WRITE, ResourcePermissionType.MANAGE_SHARING))
+ manage_share_permissions = set(
+ (ResourcePermissionType.READ,
+ ResourcePermissionType.WRITE,
+ ResourcePermissionType.MANAGE_SHARING))
current_permissions_set = set()
new_permissions_set = set()
if current_permission == ResourcePermissionType.READ:
diff --git a/django_airavata/apps/api/views.py b/django_airavata/apps/api/views.py
index 13224ff..90273bb 100644
--- a/django_airavata/apps/api/views.py
+++ b/django_airavata/apps/api/views.py
@@ -3,18 +3,25 @@ import json
import logging
import os
from datetime import datetime, timedelta
+from urllib.parse import quote
+import requests
from django.conf import settings
from django.contrib.auth.decorators import login_required
from django.core.exceptions import ObjectDoesNotExist, PermissionDenied
from django.http import FileResponse, Http404, HttpResponse, JsonResponse
from django.urls import reverse
from rest_framework import mixins
-from rest_framework.decorators import action, api_view, detail_route, list_route
+from rest_framework.decorators import (
+ action,
+ api_view,
+ detail_route,
+ list_route
+)
from rest_framework.exceptions import ParseError
from rest_framework.renderers import JSONRenderer
from rest_framework.response import Response
-from rest_framework.status import HTTP_404_NOT_FOUND, HTTP_400_BAD_REQUEST
+from rest_framework.status import HTTP_400_BAD_REQUEST, HTTP_404_NOT_FOUND
from rest_framework.views import APIView
from airavata.model.appcatalog.computeresource.ttypes import (
@@ -278,15 +285,31 @@ class ExperimentViewSet(APIBackedViewSet):
@detail_route(methods=['post'])
def launch(self, request, experiment_id=None):
try:
- experiment = request.airavata_client.getExperiment(
- self.authz_token, experiment_id)
- self._set_storage_id_and_data_dir(experiment)
- self._move_tmp_input_file_uploads_to_data_dir(experiment)
- request.airavata_client.updateExperiment(
- self.authz_token, experiment_id, experiment)
- request.airavata_client.launchExperiment(
- request.authz_token, experiment_id, self.gateway_id)
- return Response({'success': True})
+ if getattr(
+ settings,
+ 'GATEWAY_DATA_STORE_REMOTE_API',
+ None) is not None:
+ # Proxy the launch/ request to the remote Django portal
+ # instance since it must setup the experiment data directory
+ # which is only on the remote Django portal instance
+ headers = {
+ 'Authorization': f'Bearer {request.authz_token.accessToken}'}
+ r = requests.post(
+ f'{settings.GATEWAY_DATA_STORE_REMOTE_API}/experiments/{quote(experiment_id)}/launch/',
+ headers=headers,
+ )
+ r.raise_for_status()
+ return Response(r.json())
+ else:
+ experiment = request.airavata_client.getExperiment(
+ self.authz_token, experiment_id)
+ self._set_storage_id_and_data_dir(experiment)
+ self._move_tmp_input_file_uploads_to_data_dir(experiment)
+ request.airavata_client.updateExperiment(
+ self.authz_token, experiment_id, experiment)
+ request.airavata_client.launchExperiment(
+ request.authz_token, experiment_id, self.gateway_id)
+ return Response({'success': True})
except Exception as e:
return Response({'success': False, 'errorMessage': e.message})
diff --git a/django_airavata/apps/auth/views.py b/django_airavata/apps/auth/views.py
index 568d6a9..d87b031 100644
--- a/django_airavata/apps/auth/views.py
+++ b/django_airavata/apps/auth/views.py
@@ -93,6 +93,10 @@ def handle_login(request):
logger.debug("authenticated user: {}".format(user))
try:
if user is not None:
+ # Middleware will add authz_token attr to request, but since user
+ # just authenticated, authz_token won't be added yet. Login signals
+ # need the authz_token so adding it to the request now.
+ request.authz_token = utils.get_authz_token(request, user=user)
login(request, user)
if login_desktop:
return _create_login_desktop_success_response(request)
[airavata-django-portal] 05/05: AIRAVATA-3346 Handle failed bearer
token authentication
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
machristie pushed a commit to branch AIRAVATA-3346-implement-remote-fs-abstraction-of-user-storage
in repository https://gitbox.apache.org/repos/asf/airavata-django-portal.git
commit c592887583ffee67a59a0fbd494b03e634fcfdd1
Author: Marcus Christie <ma...@apache.org>
AuthorDate: Tue Sep 22 17:07:15 2020 -0400
AIRAVATA-3346 Handle failed bearer token authentication
---
django_airavata/apps/api/authentication.py | 3 +++
django_airavata/apps/auth/backends.py | 6 ++++++
2 files changed, 9 insertions(+)
diff --git a/django_airavata/apps/api/authentication.py b/django_airavata/apps/api/authentication.py
index acda113..fd282ba 100644
--- a/django_airavata/apps/api/authentication.py
+++ b/django_airavata/apps/api/authentication.py
@@ -14,6 +14,9 @@ class OAuthAuthentication(authentication.BaseAuthentication):
if 'HTTP_AUTHORIZATION' in request.META:
try:
user = authenticate(request=request)
+ if user is None:
+ raise exceptions.AuthenticationFailed(
+ "Token failed to authenticate")
_, token = request.META.get('HTTP_AUTHORIZATION').split()
# authz_token_middleware has already run, so must manually add
diff --git a/django_airavata/apps/auth/backends.py b/django_airavata/apps/auth/backends.py
index 4dd913d..1c9591d 100644
--- a/django_airavata/apps/auth/backends.py
+++ b/django_airavata/apps/auth/backends.py
@@ -39,6 +39,7 @@ class KeycloakBackend(object):
bearer, token = request.META.get('HTTP_AUTHORIZATION').split()
if bearer != "Bearer":
raise Exception("Unexpected Authorization header")
+ # implicitly validate token by using it to get userinfo
userinfo = self._get_userinfo_from_token(request, token)
# Token should be added as a request attribute (request.auth)
# self._process_token(request, token)
@@ -154,6 +155,11 @@ class KeycloakBackend(object):
oauth2_session.verify = settings.KEYCLOAK_CA_CERTFILE
userinfo = oauth2_session.get(
userinfo_url, verify=verify_ssl).json()
+ if 'error' in userinfo:
+ msg = userinfo.get('error_description')
+ if msg is None:
+ msg = f"Error fetching userinfo: {userinfo['error']}"
+ raise Exception(msg)
return userinfo
def _process_token(self, request, token):