You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by ma...@apache.org on 2021/04/15 21:15:50 UTC
[airavata-django-portal-sdk] branch mft-integration updated
(51cfd7e -> 43027ec)
This is an automated email from the ASF dual-hosted git repository.
machristie pushed a change to branch mft-integration
in repository https://gitbox.apache.org/repos/asf/airavata-django-portal-sdk.git.
from 51cfd7e AIRAVATA-3420 Implement browsing methods of MFTUserStorageProvider
new c22b2e0 AIRAVATA-3420 Implementing get_download_url, moving download view to sdk
new 43027ec AIRAVATA-3420 Fix content type determination by switching to UserStorageProvider.open
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
airavata_django_portal_sdk/urls.py | 16 ++++--
.../user_storage/__init__.py | 2 +
airavata_django_portal_sdk/user_storage/api.py | 63 +++++++++++++++-------
.../user_storage/backends/base.py | 18 ++++---
.../user_storage/backends/mft_provider.py | 40 +++++++++++---
airavata_django_portal_sdk/views.py | 57 ++++++++++++++++++++
6 files changed, 163 insertions(+), 33 deletions(-)
create mode 100644 airavata_django_portal_sdk/views.py
[airavata-django-portal-sdk] 01/02: AIRAVATA-3420 Implementing
get_download_url, moving download view to sdk
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
machristie pushed a commit to branch mft-integration
in repository https://gitbox.apache.org/repos/asf/airavata-django-portal-sdk.git
commit c22b2e0f7c86a5af35e23b4bc8aca9d30eff7b40
Author: Marcus Christie <ma...@apache.org>
AuthorDate: Thu Apr 15 14:43:05 2021 -0400
AIRAVATA-3420 Implementing get_download_url, moving download view to sdk
---
airavata_django_portal_sdk/urls.py | 16 ++++--
.../user_storage/__init__.py | 2 +
airavata_django_portal_sdk/user_storage/api.py | 25 +++++++++-
.../user_storage/backends/base.py | 18 ++++---
.../user_storage/backends/mft_provider.py | 40 ++++++++++++---
airavata_django_portal_sdk/views.py | 57 ++++++++++++++++++++++
6 files changed, 142 insertions(+), 16 deletions(-)
diff --git a/airavata_django_portal_sdk/urls.py b/airavata_django_portal_sdk/urls.py
index 9469b0f..5b70a91 100644
--- a/airavata_django_portal_sdk/urls.py
+++ b/airavata_django_portal_sdk/urls.py
@@ -1,9 +1,19 @@
+import warnings
from urllib.parse import urlencode
-from django.urls import reverse
+from django.urls import path, reverse
+
+from . import views
def get_download_url(data_product_uri):
- """Get URL for downloading data product identified by data_product_uri."""
- return (reverse("django_airavata_api:download_file") + "?" +
+ """(Deprecated) Get URL for downloading data product identified by data_product_uri."""
+ warnings.warn("Use user_storage.get_download_url instead.", DeprecationWarning)
+ return (reverse("airavata_django_portal_sdk:download_file") + "?" +
urlencode({"data-product-uri": data_product_uri}))
+
+
+app_name = 'airavata_django_portal_sdk'
+urlpatterns = [
+ path('download', views.download_file, name='download_file'),
+]
diff --git a/airavata_django_portal_sdk/user_storage/__init__.py b/airavata_django_portal_sdk/user_storage/__init__.py
index 21a22b4..283b2b9 100644
--- a/airavata_django_portal_sdk/user_storage/__init__.py
+++ b/airavata_django_portal_sdk/user_storage/__init__.py
@@ -7,6 +7,7 @@ from .api import (
dir_exists,
exists,
experiment_dir_exists,
+ get_download_url,
get_experiment_dir,
get_file,
get_file_metadata,
@@ -33,6 +34,7 @@ __all__ = [
'dir_exists',
'exists',
'experiment_dir_exists',
+ 'get_download_url',
'get_experiment_dir',
'get_file',
'get_file_metadata',
diff --git a/airavata_django_portal_sdk/user_storage/api.py b/airavata_django_portal_sdk/user_storage/api.py
index cfde16a..dec9427 100644
--- a/airavata_django_portal_sdk/user_storage/api.py
+++ b/airavata_django_portal_sdk/user_storage/api.py
@@ -7,7 +7,7 @@ import mimetypes
import os
import warnings
from http import HTTPStatus
-from urllib.parse import quote, unquote, urlparse
+from urllib.parse import quote, unquote, urlencode, urlparse
import requests
from airavata.model.data.replica.ttypes import (
@@ -21,6 +21,8 @@ from django.conf import settings
from django.core.exceptions import ObjectDoesNotExist
from ..util import convert_iso8601_to_datetime
+from django.urls import reverse
+from airavata_django_portal_sdk.user_storage.backends.base import ProvidesDownloadUrl
logger = logging.getLogger(__name__)
@@ -181,6 +183,25 @@ def move_input_file(request, data_product=None, path=None, data_product_uri=None
return move(request, data_product=data_product, path=path, data_product_uri=data_product_uri, storage_resource_id=storage_resource_id)
+def get_download_url(request, data_product=None, data_product_uri=None):
+ if data_product is None:
+ data_product = _get_data_product(request, data_product_uri)
+ if _is_remote_api():
+ raise NotImplementedError()
+ else:
+ storage_resource_id, path = _get_replica_resource_id_and_filepath(data_product)
+ backend = get_user_storage_provider(request,
+ owner_username=data_product.ownerName,
+ storage_resource_id=storage_resource_id)
+ if isinstance(backend, ProvidesDownloadUrl):
+ return backend.get_download_url(path)
+ else:
+ # if backend doesn't provide a download url, then use default one
+ # that uses backend to read the file
+ return (reverse("airavata_django_portal_sdk:download_file") + "?" +
+ urlencode({"data-product-uri": data_product.productUri}))
+
+
def open_file(request, data_product=None, data_product_uri=None):
"""
Return file object for replica if it exists in user storage. One of
@@ -435,6 +456,8 @@ def listdir(request, path, storage_resource_id=None):
mime_type = data_product.productMetadata['mime-type']
file['data-product-uri'] = data_product_uri
file['mime_type'] = mime_type
+ # TODO: remove this, there's no need for hidden files
+ file['hidden'] = False
return directories, files
diff --git a/airavata_django_portal_sdk/user_storage/backends/base.py b/airavata_django_portal_sdk/user_storage/backends/base.py
index 9936765..95601fc 100644
--- a/airavata_django_portal_sdk/user_storage/backends/base.py
+++ b/airavata_django_portal_sdk/user_storage/backends/base.py
@@ -1,4 +1,16 @@
+class ProvidesDownloadUrl:
+ """Mixin for UserStorageProvider that provides download url."""
+ def get_download_url(self, resource_path):
+ raise NotImplementedError()
+
+
+class ProvidesUploadUrl:
+ """Mixin for UserStorageProvider that provides upload url."""
+ def get_upload_url(self, resource_path):
+ raise NotImplementedError()
+
+
class UserStorageProvider:
def __init__(self, authz_token, resource_id, context=None, **kwargs):
self.authz_token = authz_token
@@ -13,15 +25,9 @@ class UserStorageProvider:
"""
raise NotImplementedError()
- def get_upload_url(self, resource_path):
- raise NotImplementedError()
-
def open(self, resource_path):
raise NotImplementedError()
- def get_download_url(self, resource_path):
- raise NotImplementedError()
-
def exists(self, resource_path):
raise NotImplementedError()
diff --git a/airavata_django_portal_sdk/user_storage/backends/mft_provider.py b/airavata_django_portal_sdk/user_storage/backends/mft_provider.py
index 488e0b0..cc53e7e 100644
--- a/airavata_django_portal_sdk/user_storage/backends/mft_provider.py
+++ b/airavata_django_portal_sdk/user_storage/backends/mft_provider.py
@@ -1,16 +1,18 @@
+import io
import logging
import os
from datetime import datetime
import grpc
+import requests
from . import CredCommon_pb2, MFTApi_pb2, MFTApi_pb2_grpc
-from .base import UserStorageProvider
+from .base import ProvidesDownloadUrl, UserStorageProvider
logger = logging.getLogger(__name__)
-class MFTUserStorageProvider(UserStorageProvider):
+class MFTUserStorageProvider(UserStorageProvider, ProvidesDownloadUrl):
def __init__(self, authz_token, resource_id, context=None, resource_token=None, mft_api_endpoint=None, mft_api_secure=False, resource_per_gateway=False, **kwargs):
super().__init__(authz_token, resource_id, context=context, **kwargs)
@@ -100,9 +102,6 @@ class MFTUserStorageProvider(UserStorageProvider):
"resource_path": d.resourcePath,
"created_time": created_time,
"size": size,
- # TODO how to handle hidden directories or directories for
- # staging input file uploads
- "hidden": False
}
)
files_data = []
@@ -127,7 +126,6 @@ class MFTUserStorageProvider(UserStorageProvider):
"resource_path": f.resourcePath,
"created_time": created_time,
"size": size,
- "hidden": False,
}
)
return directories_data, files_data
@@ -182,6 +180,36 @@ class MFTUserStorageProvider(UserStorageProvider):
logger.warning(f"Could not get metadata for {child_path} on {self.resource_id}")
return False
+ def get_download_url(self, resource_path):
+ with grpc.insecure_channel(self.mft_api_endpoint) as channel:
+ child_path = self._get_child_path(resource_path)
+ stub = MFTApi_pb2_grpc.MFTApiServiceStub(channel)
+ download_request = MFTApi_pb2.HttpDownloadApiRequest(
+ # sourceResourceId=self.resource_id,
+ # FIXME: just hacking in something to force it to work
+ sourceResourceId="remote-ssh-resource",
+ # sourcePath=child_path,
+ sourceToken=self.resource_token,
+ sourceType="SCP",
+ targetAgent="agent0",
+ mftAuthorizationToken=self.auth_token,
+ )
+ try:
+ response = stub.submitHttpDownload(download_request)
+ logger.debug(f"Download request for {self.resource_id}:{child_path}. Response = {response}")
+ return response.url
+ except Exception as e:
+ logger.error(f"submitHttpDownload request {download_request} failed.")
+ raise Exception(f"Failed to get download url for {resource_path}") from e
+
+ def open(self, resource_path):
+ download_url = self.get_download_url(resource_path)
+ r = requests.get(download_url)
+ r.raise_for_status()
+ file = io.BytesIO(r.content)
+ file.name = os.path.basename(resource_path)
+ return file
+
def _get_child_path(self, resource_path):
"""Convert possibly relative child path into absolute path."""
if not resource_path.startswith("/"):
diff --git a/airavata_django_portal_sdk/views.py b/airavata_django_portal_sdk/views.py
new file mode 100644
index 0000000..688ca84
--- /dev/null
+++ b/airavata_django_portal_sdk/views.py
@@ -0,0 +1,57 @@
+import logging
+import os
+from urllib.parse import urlparse
+
+from django.contrib.auth.decorators import login_required
+from django.core.exceptions import ObjectDoesNotExist
+from django.http import FileResponse, Http404
+from django.shortcuts import redirect
+from django.urls import reverse
+
+from airavata_django_portal_sdk import user_storage
+
+logger = logging.getLogger(__name__)
+
+
+# TODO: moving this view out of REST API means losing access token based authentication
+@login_required
+def download_file(request):
+
+ data_product_uri = request.GET.get('data-product-uri', '')
+ download_url = user_storage.get_download_url(request, data_product_uri=data_product_uri)
+ # If the download_url resolves to this view, then handle it directly
+ if urlparse(download_url).path == reverse('airavata_django_portal_sdk:download_file'):
+ return _internal_download_file(request)
+ else:
+ return redirect(download_url)
+
+
+def _internal_download_file(request):
+ data_product_uri = request.GET.get('data-product-uri', '')
+ force_download = 'download' in request.GET
+ data_product = None
+ try:
+ data_product = request.airavata_client.getDataProduct(
+ request.authz_token, data_product_uri)
+ mime_type = "application/octet-stream" # default mime-type
+ if (data_product.productMetadata and
+ 'mime-type' in data_product.productMetadata):
+ mime_type = data_product.productMetadata['mime-type']
+ # 'mime-type' url parameter overrides
+ mime_type = request.GET.get('mime-type', mime_type)
+ except Exception as e:
+ logger.warning("Failed to load DataProduct for {}"
+ .format(data_product_uri), exc_info=True)
+ raise Http404("data product does not exist") from e
+ try:
+ data_file = user_storage.open_file(request, data_product)
+ response = FileResponse(data_file, content_type=mime_type)
+ file_name = os.path.basename(data_file.name)
+ if mime_type == 'application/octet-stream' or force_download:
+ response['Content-Disposition'] = ('attachment; filename="{}"'
+ .format(file_name))
+ else:
+ response['Content-Disposition'] = f'inline; filename="{file_name}"'
+ return response
+ except ObjectDoesNotExist as e:
+ raise Http404(str(e)) from e
[airavata-django-portal-sdk] 02/02: AIRAVATA-3420 Fix content type
determination by switching to UserStorageProvider.open
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
machristie pushed a commit to branch mft-integration
in repository https://gitbox.apache.org/repos/asf/airavata-django-portal-sdk.git
commit 43027ec8c3ef755163f79b15294c5e3c16b70d18
Author: Marcus Christie <ma...@apache.org>
AuthorDate: Thu Apr 15 17:15:13 2021 -0400
AIRAVATA-3420 Fix content type determination by switching to UserStorageProvider.open
---
airavata_django_portal_sdk/user_storage/api.py | 38 ++++++++++++++------------
1 file changed, 21 insertions(+), 17 deletions(-)
diff --git a/airavata_django_portal_sdk/user_storage/api.py b/airavata_django_portal_sdk/user_storage/api.py
index dec9427..8f0f3d2 100644
--- a/airavata_django_portal_sdk/user_storage/api.py
+++ b/airavata_django_portal_sdk/user_storage/api.py
@@ -447,7 +447,8 @@ def listdir(request, path, storage_resource_id=None):
# metadata with data-product-uri and mime-type
for file in files:
data_product_uri = _get_data_product_uri(request, file['resource_path'],
- storage_resource_id=backend.resource_id)
+ storage_resource_id=backend.resource_id,
+ backend=backend)
data_product = request.airavata_client.getDataProduct(
request.authz_token, data_product_uri)
@@ -482,7 +483,8 @@ def list_experiment_dir(request, experiment_id, path="", storage_resource_id=Non
# metadata with data-product-uri and mime-type
for file in files:
data_product_uri = _get_data_product_uri(request, file['resource_path'],
- storage_resource_id=backend.resource_id)
+ storage_resource_id=backend.resource_id,
+ backend=backend)
data_product = request.airavata_client.getDataProduct(
request.authz_token, data_product_uri)
@@ -568,7 +570,7 @@ def get_rel_experiment_dir(request, experiment_id, storage_resource_id=None):
return None
-def _get_data_product_uri(request, full_path, storage_resource_id, owner=None):
+def _get_data_product_uri(request, full_path, storage_resource_id, owner=None, backend=None):
from airavata_django_portal_sdk import models
if owner is None:
@@ -578,7 +580,7 @@ def _get_data_product_uri(request, full_path, storage_resource_id, owner=None):
if user_file.exists():
product_uri = user_file[0].file_dpu
else:
- data_product = _save_data_product(request, full_path, storage_resource_id, owner=owner)
+ data_product = _save_data_product(request, full_path, storage_resource_id, owner=owner, backend=backend)
product_uri = data_product.productUri
return product_uri
@@ -588,12 +590,12 @@ def _get_data_product(request, data_product_uri):
request.authz_token, data_product_uri)
-def _save_data_product(request, full_path, storage_resource_id, name=None, content_type=None, owner=None):
+def _save_data_product(request, full_path, storage_resource_id, name=None, content_type=None, owner=None, backend=None):
"Create, register and record in DB a data product for full_path."
if owner is None:
owner = request.user.username
data_product = _create_data_product(
- owner, full_path, storage_resource_id, name=name, content_type=content_type
+ owner, full_path, storage_resource_id, name=name, content_type=content_type, backend=backend
)
product_uri = _register_data_product(request, full_path, data_product, owner=owner)
data_product.productUri = product_uri
@@ -645,7 +647,7 @@ def _delete_data_product(username, full_path):
user_file.delete()
-def _create_data_product(username, full_path, storage_resource_id, name=None, content_type=None):
+def _create_data_product(username, full_path, storage_resource_id, name=None, content_type=None, backend=None):
data_product = DataProductModel()
data_product.gatewayId = settings.GATEWAY_ID
data_product.ownerName = username
@@ -655,7 +657,7 @@ def _create_data_product(username, full_path, storage_resource_id, name=None, co
file_name = os.path.basename(full_path)
data_product.productName = file_name
data_product.dataProductType = DataProductType.FILE
- final_content_type = _determine_content_type(full_path, content_type)
+ final_content_type = _determine_content_type(full_path, content_type, backend=backend)
if final_content_type is not None:
data_product.productMetadata = {"mime-type": final_content_type}
data_replica_location = _create_replica_location(full_path, file_name, storage_resource_id)
@@ -663,20 +665,22 @@ def _create_data_product(username, full_path, storage_resource_id, name=None, co
return data_product
-def _determine_content_type(full_path, content_type=None):
+def _determine_content_type(full_path, content_type=None, backend=None):
result = content_type
if result is None:
# Try to guess the content-type from file extension
guessed_type, encoding = mimetypes.guess_type(full_path)
result = guessed_type
- # TODO: implement change to use UserStorageProvider.open to read from file
- # if result is None or result == "application/octet-stream":
- # # Check if file is Unicode text by trying to read some of it
- # try:
- # open(full_path, "r").read(1024)
- # result = "text/plain"
- # except UnicodeDecodeError:
- # logger.debug(f"Failed to read as Unicode text: {full_path}")
+ if result is None or result == "application/octet-stream":
+ # Check if file is Unicode text by trying to read some of it
+ try:
+ if backend is not None:
+ file = backend.open(full_path)
+ # Try to decode the first kb as UTF8
+ file.read(1024).decode('utf-8')
+ result = "text/plain"
+ except UnicodeDecodeError:
+ logger.debug(f"Failed to read as Unicode text: {full_path}")
return result