You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by ma...@apache.org on 2021/04/27 16:30:05 UTC
[airavata-django-portal-sdk] branch mft-integration updated
(31d2d07 -> c19c2c6)
This is an automated email from the ASF dual-hosted git repository.
machristie pushed a change to branch mft-integration
in repository https://gitbox.apache.org/repos/asf/airavata-django-portal-sdk.git.
from 31d2d07 AIRAVATA-3420 For backwards compat, keep file hidden attribute
new dc30954 AIRAVATA-3420 Fix unit tests
new c19c2c6 AIRAVATA-3420 Fixing linting, clean up
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
airavata_django_portal_sdk/user_storage/api.py | 4 +-
.../user_storage/backends/remote_api_provider.py | 0
.../user_storage_provider.py | 161 ---------------------
setup.cfg | 4 +-
tests/test_user_storage.py | 9 +-
5 files changed, 11 insertions(+), 167 deletions(-)
delete mode 100644 airavata_django_portal_sdk/user_storage/backends/remote_api_provider.py
delete mode 100644 airavata_django_portal_sdk/user_storage_provider.py
[airavata-django-portal-sdk] 02/02: AIRAVATA-3420 Fixing linting,
clean up
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
machristie pushed a commit to branch mft-integration
in repository https://gitbox.apache.org/repos/asf/airavata-django-portal-sdk.git
commit c19c2c63f18051fc69eb6868d98a97c8c93cc8a6
Author: Marcus Christie <ma...@apache.org>
AuthorDate: Tue Apr 27 12:22:51 2021 -0400
AIRAVATA-3420 Fixing linting, clean up
---
.../user_storage/backends/remote_api_provider.py | 0
.../user_storage_provider.py | 161 ---------------------
setup.cfg | 4 +-
3 files changed, 3 insertions(+), 162 deletions(-)
diff --git a/airavata_django_portal_sdk/user_storage/backends/remote_api_provider.py b/airavata_django_portal_sdk/user_storage/backends/remote_api_provider.py
deleted file mode 100644
index e69de29..0000000
diff --git a/airavata_django_portal_sdk/user_storage_provider.py b/airavata_django_portal_sdk/user_storage_provider.py
deleted file mode 100644
index b6b6842..0000000
--- a/airavata_django_portal_sdk/user_storage_provider.py
+++ /dev/null
@@ -1,161 +0,0 @@
-import logging
-import os
-from datetime import datetime
-
-import grpc
-from django.core.exceptions import ObjectDoesNotExist
-
-from . import MFTApi_pb2, MFTApi_pb2_grpc
-# from .user_storage import (TMP_INPUT_FILE_UPLOAD_DIR, _Datastore,
-# _get_data_product_uri)
-
-logger = logging.getLogger(__name__)
-
-
-class UserStorageProvider:
- def __init__(self, authz_token, context=None, *args, **kwargs):
- self.authz_token = authz_token
- self.context = context
-
- def save(self, authz_token, path, file, name=None, content_type=None):
- raise NotImplementedError()
-
- def get_upload_url(self, authz_token, path):
- raise NotImplementedError()
-
- def open(self, authz_token, resource_id=None):
- raise NotImplementedError()
-
- def get_download_url(self, authz_token, resource_id=None):
- raise NotImplementedError()
-
- def exists(self, authz_token, resource_id=None):
- raise NotImplementedError()
-
- def is_file(self, authz_token, resource_id=None):
- # TODO: is this needed if we have get_metadata?
- raise NotImplementedError()
-
- def is_dir(self, authz_token, resource_id=None):
- # TODO: is this needed if we have get_metadata?
- raise NotImplementedError()
-
- def get_metadata(self, authz_token, resource_id=None):
- raise NotImplementedError()
-
- def delete(self, authz_token, resource_id=None):
- raise NotImplementedError()
-
- def update(self, authz_token, resource_id, file):
- raise NotImplementedError()
-
-
-class FileSystemUserStorageProvider(UserStorageProvider):
- def listdir(self, request, path):
- datastore = _Datastore()
- if datastore.dir_exists(request.user.username, path):
- directories, files = datastore.list_user_dir(
- request.user.username, path)
- directories_data = []
- for d in directories:
- dpath = os.path.join(path, d)
- created_time = datastore.get_created_time(
- request.user.username, dpath)
- size = datastore.size(request.user.username, dpath)
- directories_data.append(
- {
- "name": d,
- "path": dpath,
- "created_time": created_time,
- "size": size,
- "hidden": dpath == TMP_INPUT_FILE_UPLOAD_DIR,
- }
- )
- files_data = []
- for f in files:
- user_rel_path = os.path.join(path, f)
- if not datastore.exists(request.user.username, user_rel_path):
- logger.warning(f"listdir skipping {request.user.username}:{user_rel_path}, "
- "does not exist (broken symlink?)")
- continue
- created_time = datastore.get_created_time(
- request.user.username, user_rel_path
- )
- size = datastore.size(request.user.username, user_rel_path)
- full_path = datastore.path(request.user.username, user_rel_path)
- data_product_uri = _get_data_product_uri(request, full_path)
-
- data_product = request.airavata_client.getDataProduct(
- request.authz_token, data_product_uri)
- mime_type = None
- if 'mime-type' in data_product.productMetadata:
- mime_type = data_product.productMetadata['mime-type']
- files_data.append(
- {
- "name": f,
- "path": user_rel_path,
- "data-product-uri": data_product_uri,
- "created_time": created_time,
- "mime_type": mime_type,
- "size": size,
- "hidden": False,
- }
- )
- return directories_data, files_data
- else:
- raise ObjectDoesNotExist("User storage path does not exist")
-
-
-class MFTApiUserStorageProvider(UserStorageProvider):
- def __init__(self) -> None:
- super().__init__()
-
- def dir_exists(self, path):
- channel = grpc.insecure_channel('localhost:7004')
- stub = MFTApi_pb2_grpc.MFTApiServiceStub(channel)
- request = MFTApi_pb2.FetchResourceMetadataRequest(
- resourceId="remote-ssh-dir-resource",
- resourceType="SCP",
- resourceToken="local-ssh-cred",
- resourceBackend="FILE",
- resourceCredentialBackend="FILE",
- targetAgentId="agent0",
- childPath=path,
- mftAuthorizationToken="user token")
- response = stub.getDirectoryResourceMetadata(request)
-
- def listdir(self, request, path):
- # TODO setup resourceId, etc from __init__ arguments
- channel = grpc.insecure_channel('localhost:7004')
- stub = MFTApi_pb2_grpc.MFTApiServiceStub(channel)
- request = MFTApi_pb2.FetchResourceMetadataRequest(
- resourceId="remote-ssh-dir-resource",
- resourceType="SCP",
- resourceToken="local-ssh-cred",
- resourceBackend="FILE",
- resourceCredentialBackend="FILE",
- targetAgentId="agent0",
- childPath=path,
- mftAuthorizationToken="user token")
- response = stub.getDirectoryResourceMetadata(request)
- directories_data = []
- for d in response.directories:
-
- dpath = os.path.join(path, d.friendlyName)
- created_time = datetime.fromtimestamp(d.createdTime)
- # TODO MFT API doesn't report size
- size = 0
- directories_data.append(
- {
- "name": d.friendlyName,
- "path": dpath,
- "created_time": created_time,
- "size": size,
- # TODO how to handle hidden directories or directories for
- # staging input file uploads
- "hidden": False
- }
- )
- # TODO implement
- files_data = []
- return directories_data, files_data
diff --git a/setup.cfg b/setup.cfg
index 38975e9..fe2f8f5 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,5 +1,6 @@
[flake8]
-exclude = venv
+# Exclude grpc autogenerated files
+exclude = venv, *_pb2_grpc.py, *_pb2.py
ignore = E501, W504
[isort]
@@ -13,3 +14,4 @@ ignore = E501, W504
# )
multi_line_output = 3
skip_gitignore = true
+skip_glob = *_pb2_grpc.py, *_pb2.py
[airavata-django-portal-sdk] 01/02: AIRAVATA-3420 Fix unit tests
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
machristie pushed a commit to branch mft-integration
in repository https://gitbox.apache.org/repos/asf/airavata-django-portal-sdk.git
commit dc30954baee272c438b13b3b2ed3c4a09efbd496
Author: Marcus Christie <ma...@apache.org>
AuthorDate: Tue Apr 27 12:13:39 2021 -0400
AIRAVATA-3420 Fix unit tests
---
airavata_django_portal_sdk/user_storage/api.py | 4 ++--
tests/test_user_storage.py | 9 ++++++---
2 files changed, 8 insertions(+), 5 deletions(-)
diff --git a/airavata_django_portal_sdk/user_storage/api.py b/airavata_django_portal_sdk/user_storage/api.py
index 9a45afc..16e0786 100644
--- a/airavata_django_portal_sdk/user_storage/api.py
+++ b/airavata_django_portal_sdk/user_storage/api.py
@@ -94,7 +94,7 @@ def save(request, path, file, name=None, content_type=None, storage_resource_id=
backend = get_user_storage_provider(request, storage_resource_id=storage_resource_id)
storage_resource_id, resource_path = backend.save(path, file, name=name, content_type=content_type)
data_product = _save_data_product(
- request, resource_path, storage_resource_id, name=name, content_type=content_type
+ request, resource_path, storage_resource_id, name=name, content_type=content_type, backend=backend
)
return data_product
@@ -121,7 +121,7 @@ def save_input_file(request, file, name=None, content_type=None, storage_resourc
storage_resource_id, resource_path = backend.save(
TMP_INPUT_FILE_UPLOAD_DIR, file, name=file_name)
data_product = _save_data_product(
- request, resource_path, storage_resource_id, name=name, content_type=content_type
+ request, resource_path, storage_resource_id, name=name, content_type=content_type, backend=backend
)
return data_product
diff --git a/tests/test_user_storage.py b/tests/test_user_storage.py
index a53922c..5d13b02 100644
--- a/tests/test_user_storage.py
+++ b/tests/test_user_storage.py
@@ -11,6 +11,7 @@ from airavata.model.data.replica.ttypes import (
DataReplicaLocationModel,
ReplicaLocationCategory
)
+from airavata.model.security.ttypes import AuthzToken
from django.contrib.auth.models import User
from django.test import RequestFactory, TestCase, override_settings
@@ -32,7 +33,9 @@ class BaseTestCase(TestCase):
self.product_uri = f"airavata-dp://{uuid.uuid4()}"
self.request.airavata_client.registerDataProduct.return_value = \
self.product_uri
- self.request.authz_token = "dummy"
+ self.request.authz_token = AuthzToken(accessToken="dummy",
+ claimsMap={'gatewayID': GATEWAY_ID,
+ 'userName': self.user.username})
class SaveTests(BaseTestCase):
@@ -60,7 +63,7 @@ class SaveTests(BaseTestCase):
self.assertDictEqual({'mime-type': 'text/plain'},
dp.productMetadata)
self.assertEqual(1, len(dp.replicaLocations))
- self.assertEqual(f"file://gateway.com:{path}/{file.name}",
+ self.assertEqual(f"{path}/{file.name}",
dp.replicaLocations[0].filePath)
def test_save_with_name_and_content_type(self):
@@ -90,7 +93,7 @@ class SaveTests(BaseTestCase):
self.assertDictEqual({'mime-type': 'application/some-app'},
dp.productMetadata)
self.assertEqual(1, len(dp.replicaLocations))
- self.assertEqual(f"file://gateway.com:{path}/bar.txt",
+ self.assertEqual(f"{path}/bar.txt",
dp.replicaLocations[0].filePath)
def test_save_with_unknown_text_file_type(self):