You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by ma...@apache.org on 2021/04/15 21:15:51 UTC
[airavata-django-portal-sdk] 01/02: AIRAVATA-3420 Implementing
get_download_url, moving download view to sdk
This is an automated email from the ASF dual-hosted git repository.
machristie pushed a commit to branch mft-integration
in repository https://gitbox.apache.org/repos/asf/airavata-django-portal-sdk.git
commit c22b2e0f7c86a5af35e23b4bc8aca9d30eff7b40
Author: Marcus Christie <ma...@apache.org>
AuthorDate: Thu Apr 15 14:43:05 2021 -0400
AIRAVATA-3420 Implementing get_download_url, moving download view to sdk
---
airavata_django_portal_sdk/urls.py | 16 ++++--
.../user_storage/__init__.py | 2 +
airavata_django_portal_sdk/user_storage/api.py | 25 +++++++++-
.../user_storage/backends/base.py | 18 ++++---
.../user_storage/backends/mft_provider.py | 40 ++++++++++++---
airavata_django_portal_sdk/views.py | 57 ++++++++++++++++++++++
6 files changed, 142 insertions(+), 16 deletions(-)
diff --git a/airavata_django_portal_sdk/urls.py b/airavata_django_portal_sdk/urls.py
index 9469b0f..5b70a91 100644
--- a/airavata_django_portal_sdk/urls.py
+++ b/airavata_django_portal_sdk/urls.py
@@ -1,9 +1,19 @@
+import warnings
from urllib.parse import urlencode
-from django.urls import reverse
+from django.urls import path, reverse
+
+from . import views
def get_download_url(data_product_uri):
- """Get URL for downloading data product identified by data_product_uri."""
- return (reverse("django_airavata_api:download_file") + "?" +
+ """(Deprecated) Get URL for downloading data product identified by data_product_uri."""
+ warnings.warn("Use user_storage.get_download_url instead.", DeprecationWarning)
+ return (reverse("airavata_django_portal_sdk:download_file") + "?" +
urlencode({"data-product-uri": data_product_uri}))
+
+
+app_name = 'airavata_django_portal_sdk'
+urlpatterns = [
+ path('download', views.download_file, name='download_file'),
+]
diff --git a/airavata_django_portal_sdk/user_storage/__init__.py b/airavata_django_portal_sdk/user_storage/__init__.py
index 21a22b4..283b2b9 100644
--- a/airavata_django_portal_sdk/user_storage/__init__.py
+++ b/airavata_django_portal_sdk/user_storage/__init__.py
@@ -7,6 +7,7 @@ from .api import (
dir_exists,
exists,
experiment_dir_exists,
+ get_download_url,
get_experiment_dir,
get_file,
get_file_metadata,
@@ -33,6 +34,7 @@ __all__ = [
'dir_exists',
'exists',
'experiment_dir_exists',
+ 'get_download_url',
'get_experiment_dir',
'get_file',
'get_file_metadata',
diff --git a/airavata_django_portal_sdk/user_storage/api.py b/airavata_django_portal_sdk/user_storage/api.py
index cfde16a..dec9427 100644
--- a/airavata_django_portal_sdk/user_storage/api.py
+++ b/airavata_django_portal_sdk/user_storage/api.py
@@ -7,7 +7,7 @@ import mimetypes
import os
import warnings
from http import HTTPStatus
-from urllib.parse import quote, unquote, urlparse
+from urllib.parse import quote, unquote, urlencode, urlparse
import requests
from airavata.model.data.replica.ttypes import (
@@ -21,6 +21,8 @@ from django.conf import settings
from django.core.exceptions import ObjectDoesNotExist
from ..util import convert_iso8601_to_datetime
+from django.urls import reverse
+from airavata_django_portal_sdk.user_storage.backends.base import ProvidesDownloadUrl
logger = logging.getLogger(__name__)
@@ -181,6 +183,25 @@ def move_input_file(request, data_product=None, path=None, data_product_uri=None
return move(request, data_product=data_product, path=path, data_product_uri=data_product_uri, storage_resource_id=storage_resource_id)
+def get_download_url(request, data_product=None, data_product_uri=None):
+ if data_product is None:
+ data_product = _get_data_product(request, data_product_uri)
+ if _is_remote_api():
+ raise NotImplementedError()
+ else:
+ storage_resource_id, path = _get_replica_resource_id_and_filepath(data_product)
+ backend = get_user_storage_provider(request,
+ owner_username=data_product.ownerName,
+ storage_resource_id=storage_resource_id)
+ if isinstance(backend, ProvidesDownloadUrl):
+ return backend.get_download_url(path)
+ else:
+ # if backend doesn't provide a download url, then use default one
+ # that uses backend to read the file
+ return (reverse("airavata_django_portal_sdk:download_file") + "?" +
+ urlencode({"data-product-uri": data_product.productUri}))
+
+
def open_file(request, data_product=None, data_product_uri=None):
"""
Return file object for replica if it exists in user storage. One of
@@ -435,6 +456,8 @@ def listdir(request, path, storage_resource_id=None):
mime_type = data_product.productMetadata['mime-type']
file['data-product-uri'] = data_product_uri
file['mime_type'] = mime_type
+ # TODO: remove this, there's no need for hidden files
+ file['hidden'] = False
return directories, files
diff --git a/airavata_django_portal_sdk/user_storage/backends/base.py b/airavata_django_portal_sdk/user_storage/backends/base.py
index 9936765..95601fc 100644
--- a/airavata_django_portal_sdk/user_storage/backends/base.py
+++ b/airavata_django_portal_sdk/user_storage/backends/base.py
@@ -1,4 +1,16 @@
+class ProvidesDownloadUrl:
+ """Mixin for UserStorageProvider that provides download url."""
+ def get_download_url(self, resource_path):
+ raise NotImplementedError()
+
+
+class ProvidesUploadUrl:
+ """Mixin for UserStorageProvider that provides upload url."""
+ def get_upload_url(self, resource_path):
+ raise NotImplementedError()
+
+
class UserStorageProvider:
def __init__(self, authz_token, resource_id, context=None, **kwargs):
self.authz_token = authz_token
@@ -13,15 +25,9 @@ class UserStorageProvider:
"""
raise NotImplementedError()
- def get_upload_url(self, resource_path):
- raise NotImplementedError()
-
def open(self, resource_path):
raise NotImplementedError()
- def get_download_url(self, resource_path):
- raise NotImplementedError()
-
def exists(self, resource_path):
raise NotImplementedError()
diff --git a/airavata_django_portal_sdk/user_storage/backends/mft_provider.py b/airavata_django_portal_sdk/user_storage/backends/mft_provider.py
index 488e0b0..cc53e7e 100644
--- a/airavata_django_portal_sdk/user_storage/backends/mft_provider.py
+++ b/airavata_django_portal_sdk/user_storage/backends/mft_provider.py
@@ -1,16 +1,18 @@
+import io
import logging
import os
from datetime import datetime
import grpc
+import requests
from . import CredCommon_pb2, MFTApi_pb2, MFTApi_pb2_grpc
-from .base import UserStorageProvider
+from .base import ProvidesDownloadUrl, UserStorageProvider
logger = logging.getLogger(__name__)
-class MFTUserStorageProvider(UserStorageProvider):
+class MFTUserStorageProvider(UserStorageProvider, ProvidesDownloadUrl):
def __init__(self, authz_token, resource_id, context=None, resource_token=None, mft_api_endpoint=None, mft_api_secure=False, resource_per_gateway=False, **kwargs):
super().__init__(authz_token, resource_id, context=context, **kwargs)
@@ -100,9 +102,6 @@ class MFTUserStorageProvider(UserStorageProvider):
"resource_path": d.resourcePath,
"created_time": created_time,
"size": size,
- # TODO how to handle hidden directories or directories for
- # staging input file uploads
- "hidden": False
}
)
files_data = []
@@ -127,7 +126,6 @@ class MFTUserStorageProvider(UserStorageProvider):
"resource_path": f.resourcePath,
"created_time": created_time,
"size": size,
- "hidden": False,
}
)
return directories_data, files_data
@@ -182,6 +180,36 @@ class MFTUserStorageProvider(UserStorageProvider):
logger.warning(f"Could not get metadata for {child_path} on {self.resource_id}")
return False
+ def get_download_url(self, resource_path):
+ with grpc.insecure_channel(self.mft_api_endpoint) as channel:
+ child_path = self._get_child_path(resource_path)
+ stub = MFTApi_pb2_grpc.MFTApiServiceStub(channel)
+ download_request = MFTApi_pb2.HttpDownloadApiRequest(
+ # sourceResourceId=self.resource_id,
+ # FIXME: just hacking in something to force it to work
+ sourceResourceId="remote-ssh-resource",
+ # sourcePath=child_path,
+ sourceToken=self.resource_token,
+ sourceType="SCP",
+ targetAgent="agent0",
+ mftAuthorizationToken=self.auth_token,
+ )
+ try:
+ response = stub.submitHttpDownload(download_request)
+ logger.debug(f"Download request for {self.resource_id}:{child_path}. Response = {response}")
+ return response.url
+ except Exception as e:
+ logger.error(f"submitHttpDownload request {download_request} failed.")
+ raise Exception(f"Failed to get download url for {resource_path}") from e
+
+ def open(self, resource_path):
+ download_url = self.get_download_url(resource_path)
+ r = requests.get(download_url)
+ r.raise_for_status()
+ file = io.BytesIO(r.content)
+ file.name = os.path.basename(resource_path)
+ return file
+
def _get_child_path(self, resource_path):
"""Convert possibly relative child path into absolute path."""
if not resource_path.startswith("/"):
diff --git a/airavata_django_portal_sdk/views.py b/airavata_django_portal_sdk/views.py
new file mode 100644
index 0000000..688ca84
--- /dev/null
+++ b/airavata_django_portal_sdk/views.py
@@ -0,0 +1,57 @@
+import logging
+import os
+from urllib.parse import urlparse
+
+from django.contrib.auth.decorators import login_required
+from django.core.exceptions import ObjectDoesNotExist
+from django.http import FileResponse, Http404
+from django.shortcuts import redirect
+from django.urls import reverse
+
+from airavata_django_portal_sdk import user_storage
+
+logger = logging.getLogger(__name__)
+
+
+# TODO: moving this view out of REST API means losing access token based authentication
+@login_required
+def download_file(request):
+
+ data_product_uri = request.GET.get('data-product-uri', '')
+ download_url = user_storage.get_download_url(request, data_product_uri=data_product_uri)
+ # If the download_url resolves to this view, then handle it directly
+ if urlparse(download_url).path == reverse('airavata_django_portal_sdk:download_file'):
+ return _internal_download_file(request)
+ else:
+ return redirect(download_url)
+
+
+def _internal_download_file(request):
+ data_product_uri = request.GET.get('data-product-uri', '')
+ force_download = 'download' in request.GET
+ data_product = None
+ try:
+ data_product = request.airavata_client.getDataProduct(
+ request.authz_token, data_product_uri)
+ mime_type = "application/octet-stream" # default mime-type
+ if (data_product.productMetadata and
+ 'mime-type' in data_product.productMetadata):
+ mime_type = data_product.productMetadata['mime-type']
+ # 'mime-type' url parameter overrides
+ mime_type = request.GET.get('mime-type', mime_type)
+ except Exception as e:
+ logger.warning("Failed to load DataProduct for {}"
+ .format(data_product_uri), exc_info=True)
+ raise Http404("data product does not exist") from e
+ try:
+ data_file = user_storage.open_file(request, data_product)
+ response = FileResponse(data_file, content_type=mime_type)
+ file_name = os.path.basename(data_file.name)
+ if mime_type == 'application/octet-stream' or force_download:
+ response['Content-Disposition'] = ('attachment; filename="{}"'
+ .format(file_name))
+ else:
+ response['Content-Disposition'] = f'inline; filename="{file_name}"'
+ return response
+ except ObjectDoesNotExist as e:
+ raise Http404(str(e)) from e